CMU MOSI Gated RNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of AEY 3.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "nranlSupmdn6",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "833f9118-b7e6-4a39-b811-d5a10f07267f"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8fb98421-644a-42b5-bf07-4dda4494c5d9\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-8fb98421-644a-42b5-bf07-4dda4494c5d9\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving audio.pickle to audio.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8J8o6eonovjv",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "a10bea66-a101-4c98-bc73-81829fbd9e48"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-b0a69c4e-aaf0-4a17-a7b6-ddf8e5beb66d\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-b0a69c4e-aaf0-4a17-a7b6-ddf8e5beb66d\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving audio_2way.pickle to audio_2way.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2Nb5iY6ozCr",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "5cb4e97c-d886-4d63-d6e7-4a5947f68ee3"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-570b2ff2-8b5d-46ee-9736-3271b1854eb5\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-570b2ff2-8b5d-46ee-9736-3271b1854eb5\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving text.pickle to text.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCBJseGLoyFg",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "f6891604-2d49-4791-f6f6-68ec3a1ceaa8"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-0400ee24-e163-44d6-a37a-7118687e5f0f\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-0400ee24-e163-44d6-a37a-7118687e5f0f\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving text_2way.pickle to text_2way.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JeT2DlzhoxZA",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "c4735ea8-075a-4838-a100-50ce0a9f23bd"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-a93bd153-1047-49e7-bfb8-5b4a3458c8eb\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-a93bd153-1047-49e7-bfb8-5b4a3458c8eb\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving video.pickle to video.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e7MFpmTMoYIb",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "ef0adf73-0f31-405e-cb0d-b80dfc780dfb"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-94887f80-2bc1-4161-b5e3-4e6cee7cafbf\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-94887f80-2bc1-4161-b5e3-4e6cee7cafbf\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving video_2way.pickle to video_2way.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVqz0H0Mn1Um",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "545706df-5448-4629-b5e6-ec47195bafe9"
      },
      "source": [
        "!unzip raw.rar"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  raw.rar\n",
            "  End-of-central-directory signature not found.  Either this file is not\n",
            "  a zipfile, or it constitutes one disk of a multi-part archive.  In the\n",
            "  latter case the central directory and zipfile comment will be found on\n",
            "  the last disk(s) of this archive.\n",
            "unzip:  cannot find zipfile directory in one of raw.rar or\n",
            "        raw.rar.zip, and cannot find raw.rar.ZIP, period.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kSAW_OQvmbz8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8df9b992-a566-453e-c0f8-147fd0575802"
      },
      "source": [
        "import pickle\n",
        "import sys\n",
        "import keras\n",
        "import tqdm\n",
        "import argparse\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tqdm import tqdm\n",
        "from sklearn.metrics import f1_score\n",
        "from tensorflow.python.layers.core import Dense"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VETF6KbKmb0l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def createOneHot(train_label, test_label):\n",
        "    print(\"train_label shape\", train_label.shape)\n",
        "    print(\"test_label shape\", test_label.shape)\n",
        "    maxlen = int(max(train_label.max(), test_label.max()))\n",
        "    print(train_label.max())\n",
        "    print( test_label.max())\n",
        "    print(\"maxlen\", maxlen)\n",
        "    train = np.zeros((train_label.shape[0], train_label.shape[1], maxlen + 1))\n",
        "    test = np.zeros((test_label.shape[0], test_label.shape[1], maxlen + 1))\n",
        "\n",
        "    for i in range(train_label.shape[0]):\n",
        "        for j in range(train_label.shape[1]):\n",
        "            train[i, j, train_label[i, j]] = 1\n",
        "    print(\"train\", train)\n",
        "    for i in range(test_label.shape[0]):\n",
        "        for j in range(test_label.shape[1]):\n",
        "            test[i, j, test_label[i, j]] = 1\n",
        "    print(\"test\", test)\n",
        "    return train, test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rasj09fCmb04",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def batch_iter(data, batch_size, shuffle=True):\n",
        "    \"\"\"\n",
        "    Generates a batch iterator for a dataset.\n",
        "    \"\"\"\n",
        "    data = np.array(data)\n",
        "    data_size = len(data)\n",
        "    num_batches_per_epoch = int((len(data) - 1) / batch_size) + 1\n",
        "    # Shuffle the data at each epoch\n",
        "    if shuffle:\n",
        "        shuffle_indices = np.random.permutation(np.arange(data_size))\n",
        "        shuffled_data = data[shuffle_indices]\n",
        "    else:\n",
        "        shuffled_data = data\n",
        "    for batch_num in range(num_batches_per_epoch):\n",
        "        start_index = batch_num * batch_size\n",
        "        end_index = min((batch_num + 1) * batch_size, data_size)\n",
        "        yield shuffled_data[start_index:end_index]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNLJtJY4mb1G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_raw_data(data, classes):\n",
        "    mode = 'audio'\n",
        "    with open('{1}_{2}way.pickle'.format(data, mode, classes), 'rb') as handle:\n",
        "        u = pickle._Unpickler(handle)\n",
        "        u.encoding = 'latin1'\n",
        "        if data == 'mosi':\n",
        "            (audio_train, train_label, audio_test, test_label, _, train_length, test_length) = u.load()\n",
        "    mode = 'text'\n",
        "    with open('{1}_{2}way.pickle'.format(data, mode, classes), 'rb') as handle:\n",
        "        u = pickle._Unpickler(handle)\n",
        "        u.encoding = 'latin1'\n",
        "        if data == 'mosi':\n",
        "            (text_train, train_label, text_test, test_label, _, train_length, test_length) = u.load()\n",
        "        \n",
        "    mode = 'video'\n",
        "    with open('{1}_{2}way.pickle'.format(data, mode, classes), 'rb') as handle:\n",
        "        u = pickle._Unpickler(handle)\n",
        "        u.encoding = 'latin1'\n",
        "        if data == 'mosi':\n",
        "            (video_train, train_label, video_test, test_label, _, train_length, test_length) = u.load()\n",
        "        print(\"DATA\", video_train)\n",
        "        print(\"shape\",video_train.shape)\n",
        "\n",
        "    train_data = np.concatenate((audio_train, video_train, text_train), axis=-1)\n",
        "    test_data = np.concatenate((audio_test, video_test, text_test), axis=-1)\n",
        "\n",
        "    train_label = train_label.astype('int')\n",
        "    test_label = test_label.astype('int')\n",
        "    print(train_data.shape)\n",
        "    print(test_data.shape)\n",
        "    train_mask = np.zeros((train_data.shape[0], train_data.shape[1]), dtype='float')\n",
        "    for i in range(len(train_length)):\n",
        "        train_mask[i, :train_length[i]] = 1.0\n",
        "\n",
        "    test_mask = np.zeros((test_data.shape[0], test_data.shape[1]), dtype='float')\n",
        "    for i in range(len(test_length)):\n",
        "        test_mask[i, :test_length[i]] = 1.0\n",
        "\n",
        "    train_label, test_label = createOneHot(train_label, test_label)\n",
        "\n",
        "    print('train_mask', train_mask.shape)\n",
        "\n",
        "    seqlen_train = train_length\n",
        "    seqlen_test = test_length\n",
        "\n",
        "    return train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NI37UJhDmb1T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTM_Model():\n",
        "    def __init__(self, input_shape, lr, a_dim, v_dim, t_dim, emotions, attn_fusion=True, unimodal=False,\n",
        "                 enable_attn_2=False, seed=1234):\n",
        "        if unimodal:\n",
        "            self.input = tf.placeholder(dtype=tf.float32, shape=(None, input_shape[0], input_shape[1]))\n",
        "        else:\n",
        "            self.a_input = tf.placeholder(dtype=tf.float32, shape=(None, input_shape[0], a_dim))\n",
        "            self.v_input = tf.placeholder(dtype=tf.float32, shape=(None, input_shape[0], v_dim))\n",
        "            self.t_input = tf.placeholder(dtype=tf.float32, shape=(None, input_shape[0], t_dim))\n",
        "        self.emotions = emotions\n",
        "        self.mask = tf.placeholder(dtype=tf.float32, shape=(None, input_shape[0]))\n",
        "        self.seq_len = tf.placeholder(tf.int32, [None, ], name=\"seq_len\")\n",
        "        self.y = tf.placeholder(tf.int32, [None, input_shape[0], self.emotions], name=\"y\")\n",
        "        self.lr = lr\n",
        "        self.seed = seed\n",
        "        self.attn_fusion = attn_fusion\n",
        "        self.unimodal = unimodal\n",
        "        self.lstm_dropout = tf.placeholder(tf.float32, name=\"lstm_dropout\")\n",
        "        self.dropout = tf.placeholder(tf.float32, name=\"dropout\")\n",
        "        self.lstm_inp_dropout = tf.placeholder(tf.float32, name=\"lstm_inp_dropout\")\n",
        "        self.dropout_lstm_out = tf.placeholder(tf.float32, name=\"dropout_lstm_out\")\n",
        "        self.attn_2 = enable_attn_2\n",
        "\n",
        "        # Build the model\n",
        "        self._build_model_op()\n",
        "        self._initialize_optimizer()\n",
        "\n",
        "    def BiGRU(self, inputs, output_size, name, dropout_keep_rate):\n",
        "        with tf.variable_scope('rnn_' + name, reuse=tf.AUTO_REUSE):\n",
        "            kernel_init = tf.glorot_uniform_initializer(seed=self.seed, dtype=tf.float32)\n",
        "            bias_init = tf.zeros_initializer()\n",
        "\n",
        "#             fw_cell = tf.contrib.rnn.GLSTMCell(output_size, name='gru', reuse=tf.AUTO_REUSE, activation=tf.nn.tanh,\n",
        "#                                              kernel_initializer=kernel_init, bias_initializer=bias_init)\n",
        "#             fw_cell = tf.contrib.rnn.DropoutWrapper(fw_cell, output_keep_prob=dropout_keep_rate)\n",
        "            fw_cell = tf.contrib.rnn.GRUBlockCellV2(output_size, reuse=tf.AUTO_REUSE)\n",
        "            fw_cell = tf.contrib.rnn.DropoutWrapper(fw_cell, output_keep_prob=dropout_keep_rate)\n",
        "            # bw_cell = tf.contrib.rnn.GRUCell(output_size, name='gru', reuse=tf.AUTO_REUSE, activation=tf.nn.tanh,\n",
        "            #                                 kernel_initializer=kernel_init, bias_initializer=bias_init)\n",
        "            # bw_cell = tf.contrib.rnn.DropoutWrapper(bw_cell, output_keep_prob=dropout_keep_rate)\n",
        "\n",
        "            outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell, cell_bw=fw_cell, inputs=inputs,\n",
        "                                                         sequence_length=self.seq_len, dtype=tf.float32)\n",
        "\n",
        "            output_fw, output_bw = outputs\n",
        "            output = tf.concat([output_fw, output_bw], axis=-1)\n",
        "            return output\n",
        "\n",
        "    def self_attention(self, inputs_a, inputs_v, inputs_t, name):\n",
        "        \"\"\"\n",
        "        :param inputs_a: audio input (B, T, dim)\n",
        "        :param inputs_v: video input (B, T, dim)\n",
        "        :param inputs_t: text input (B, T, dim)\n",
        "        :param name: scope name\n",
        "        :return:\n",
        "        \"\"\"\n",
        "\n",
        "        inputs_a = tf.expand_dims(inputs_a, axis=1)\n",
        "        inputs_v = tf.expand_dims(inputs_v, axis=1)\n",
        "        inputs_t = tf.expand_dims(inputs_t, axis=1)\n",
        "        # inputs = (B, 3, T, dim)\n",
        "        inputs = tf.concat([inputs_a, inputs_v, inputs_t], axis=1)\n",
        "        t = inputs.get_shape()[2].value\n",
        "        share_param = True\n",
        "        hidden_size = inputs.shape[-1].value  # D value - hidden size of the RNN layer\n",
        "        kernel_init1 = tf.glorot_uniform_initializer(seed=self.seed, dtype=tf.float32)\n",
        "        # kernel_init2 = tf.random_normal_initializer(seed=self.seed, dtype=tf.float32,stddev=0.01)\n",
        "        # bias_init = tf.zeros_initializer()\n",
        "        dense = Dense(hidden_size, kernel_initializer=kernel_init1)\n",
        "        if share_param:\n",
        "            scope_name = 'self_attn'\n",
        "        else:\n",
        "            scope_name = 'self_attn' + name\n",
        "        # print(scope_name)\n",
        "        inputs = tf.transpose(inputs, [2, 0, 1, 3])\n",
        "        with tf.variable_scope(scope_name):\n",
        "            outputs = []\n",
        "            for x in range(t):\n",
        "                t_x = inputs[x, :, :, :]\n",
        "                # t_x => B, 3, dim\n",
        "                den = True\n",
        "                if den:\n",
        "                    x_proj = dense(t_x)\n",
        "                    x_proj = tf.nn.relu(x_proj)\n",
        "                else:\n",
        "                    x_proj = t_x\n",
        "                u_w = tf.Variable(tf.random_normal([hidden_size, 1], stddev=0.01, seed=1234))\n",
        "                x = tf.tensordot(x_proj, u_w, axes=1)\n",
        "                alphas = tf.nn.softmax(x, axis=-1)\n",
        "                output = tf.matmul(tf.transpose(t_x, [0, 2, 1]), alphas)\n",
        "                output = tf.squeeze(output, -1)\n",
        "                outputs.append(output)\n",
        "\n",
        "            final_output = tf.stack(outputs, axis=1)\n",
        "            # print('final_output', final_output.get_shape())\n",
        "            return final_output\n",
        "\n",
        "    def attention(self, inputs_a, inputs_b, attention_size, params, mask=None, return_alphas=False):\n",
        "        \"\"\"\n",
        "        inputs_a = (b, 18, 100)\n",
        "        inputs_b = (b, 100)\n",
        "        :param inputs_a:\n",
        "        :param inputs_b:\n",
        "        :param attention_size:\n",
        "        :param time_major:\n",
        "        :param return_alphas:\n",
        "        :return:\n",
        "        \"\"\"\n",
        "        if mask is not None:\n",
        "            mask = tf.cast(self.mask, tf.bool)\n",
        "        shared = True\n",
        "        if shared:\n",
        "            scope_name = 'attn'\n",
        "        else:\n",
        "            scope_name = 'attn_'\n",
        "        with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE):\n",
        "            hidden_size = inputs_a.shape[2].value  # D value - hidden size of the RNN layer\n",
        "            den = False\n",
        "            x_proj = inputs_a\n",
        "            y_proj = inputs_b\n",
        "            # print('x_proj', x_proj.get_shape())\n",
        "            # print('y_proj', y_proj.get_shape())\n",
        "\n",
        "            # Trainable parameters\n",
        "            w_omega = params['w_omega']\n",
        "            b_omega = params['b_omega']\n",
        "            # dense_attention_2 = params['dense']\n",
        "            with tf.variable_scope('v', reuse=tf.AUTO_REUSE):\n",
        "                # Applying fully connected layer with non-linear activation to each of the B*T timestamps;\n",
        "                #  the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size\n",
        "\n",
        "                v = tf.tensordot(x_proj, w_omega, axes=1) + b_omega\n",
        "                # v  = dense_attention_2(x_proj)\n",
        "\n",
        "            # For each of the timestamps its vector of size A from `v` is reduced with `u` vector\n",
        "            vu = tf.tanh(tf.matmul(v, tf.expand_dims(y_proj, -1), name='vu'))  # (B,T) shape (B T A) * (B A 1) = (B T)\n",
        "            vu = tf.squeeze(vu, -1)\n",
        "            # print('vu', vu.get_shape())\n",
        "            # masking\n",
        "            # mask = None\n",
        "            if mask is not None:\n",
        "                vu = tf.where(mask, vu, tf.zeros(tf.shape(vu), dtype=tf.float32))\n",
        "\n",
        "            alphas = tf.nn.softmax(vu, 1, name='alphas')  # (B,T) shape\n",
        "            if mask is not None:\n",
        "                alphas = tf.where(mask, alphas, tf.zeros(tf.shape(alphas), dtype=tf.float32))\n",
        "                a = tf.reduce_sum(tf.expand_dims(alphas, -1), axis=1)\n",
        "                condition = tf.equal(a, 0.0)\n",
        "                case_true = tf.ones(tf.shape(a), tf.float32)\n",
        "                a_m = tf.where(condition, case_true, a)\n",
        "                alphas = tf.divide(alphas, a_m)\n",
        "\n",
        "            # print('alphas', alphas.get_shape())\n",
        "\n",
        "            # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape\n",
        "            output = tf.matmul(tf.transpose(inputs_a, [0, 2, 1]), tf.expand_dims(alphas, -1))\n",
        "            output = tf.squeeze(output, -1)\n",
        "            # print('r', output.get_shape())\n",
        "            # output = tf.reduce_sum(r, 1)\n",
        "\n",
        "            if not return_alphas:\n",
        "                return tf.expand_dims(output, 1)\n",
        "            else:\n",
        "                return tf.expand_dims(output, 1), alphas\n",
        "\n",
        "    def self_attention_2(self, inputs, name):\n",
        "        \"\"\"\n",
        "        :param inputs_a: audio input (B, T, dim)\n",
        "        :param inputs_v: video input (B, T, dim)\n",
        "        :param inputs_t: text input (B, T, dim)\n",
        "        :param name: scope name\n",
        "        :return:\n",
        "        \"\"\"\n",
        "\n",
        "        t = inputs.get_shape()[1].value\n",
        "        share_param = True\n",
        "        hidden_size = inputs.shape[-1].value  # D value - hidden size of the RNN layer\n",
        "        if share_param:\n",
        "            scope_name = 'self_attn_2'\n",
        "        else:\n",
        "            scope_name = 'self_attn_2' + name\n",
        "        # print(scope_name)\n",
        "        # inputs = tf.transpose(inputs, [2, 0, 1, 3])\n",
        "        # dense = Dense(hidden_size)\n",
        "        # init1 = tf.random_normal_initializer(seed=self.seed, dtype=tf.float32,stddev=0.01)\n",
        "        attention_size = hidden_size\n",
        "        w_omega = tf.Variable(tf.random_normal([hidden_size, attention_size], stddev=0.01, seed=self.seed))\n",
        "        b_omega = tf.Variable(tf.random_normal([attention_size], stddev=0.01, seed=self.seed))\n",
        "        # dense_attention_2 = Dense(attention_size, activation=None,kernel_initializer=init1,\n",
        "        #kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))\n",
        "        params = {'w_omega': w_omega,\n",
        "                  'b_omega': b_omega,\n",
        "                  # 'dense': dense_attention_2\n",
        "                  }\n",
        "        with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE):\n",
        "            outputs = []\n",
        "            for x in range(t):\n",
        "                t_x = inputs[:, x, :]\n",
        "\n",
        "                output = self.attention(inputs, t_x, hidden_size, params, self.mask)  # (b, d)\n",
        "                outputs.append(output)\n",
        "\n",
        "            final_output = tf.concat(outputs, axis=1)\n",
        "            return final_output\n",
        "\n",
        "    def _build_model_op(self):\n",
        "        # self attention\n",
        "        if self.unimodal:\n",
        "            input = self.input\n",
        "        else:\n",
        "            if self.attn_fusion:\n",
        "                input = self.self_attention(self.a_input, self.v_input, self.t_input, '')\n",
        "                input = input * tf.expand_dims(self.mask, axis=-1)\n",
        "            else:\n",
        "                input = tf.concat([self.a_input, self.v_input, self.t_input], axis=-1)\n",
        "\n",
        "        # input = tf.nn.dropout(input, 1-self.lstm_inp_dropout)\n",
        "        self.gru_output = self.BiGRU(input, 100, 'gru', 1 - self.lstm_dropout)\n",
        "        self.inter = tf.nn.dropout(self.gru_output, 1 - self.dropout_lstm_out)\n",
        "        # self.inter = self.gru_output\n",
        "        if self.attn_2:\n",
        "            self.inter = self.self_attention_2(self.inter, '')\n",
        "        init = tf.glorot_uniform_initializer(seed=self.seed, dtype=tf.float32)\n",
        "        if self.unimodal:\n",
        "            self.inter1 = Dense(100, activation=tf.nn.relu,\n",
        "                                kernel_initializer=init, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))(\n",
        "                self.inter)\n",
        "        else:\n",
        "            self.inter1 = Dense(200, activation=tf.nn.relu,\n",
        "                                kernel_initializer=init, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))(\n",
        "                self.inter)\n",
        "            self.inter1 = self.inter1 * tf.expand_dims(self.mask, axis=-1)\n",
        "            self.inter1 = Dense(200, activation=tf.nn.relu,\n",
        "                                kernel_initializer=init, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))(\n",
        "                self.inter1)\n",
        "            self.inter1 = self.inter1 * tf.expand_dims(self.mask, axis=-1)\n",
        "            self.inter1 = Dense(200, activation=tf.nn.relu,\n",
        "                                kernel_initializer=init, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))(\n",
        "                self.inter1)\n",
        "        self.inter1 = self.inter1 * tf.expand_dims(self.mask, axis=-1)\n",
        "        self.inter1 = tf.nn.dropout(self.inter1, 1 - self.dropout)\n",
        "        self.output = Dense(self.emotions, kernel_initializer=init,\n",
        "                            kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001))(self.inter1)\n",
        "        # print('self.output', self.output.get_shape())\n",
        "        self.preds = tf.nn.softmax(self.output)\n",
        "        # To calculate the number correct, we want to count padded steps as incorrect\n",
        "        correct = tf.cast(\n",
        "            tf.equal(tf.argmax(self.preds, -1, output_type=tf.int32), tf.argmax(self.y, -1, output_type=tf.int32)),\n",
        "            tf.int32) * tf.cast(self.mask, tf.int32)\n",
        "\n",
        "        # To calculate accuracy we want to divide by the number of non-padded time-steps,\n",
        "        # rather than taking the mean\n",
        "        self.accuracy = tf.reduce_sum(tf.cast(correct, tf.float32)) / tf.reduce_sum(tf.cast(self.seq_len, tf.float32))\n",
        "        # y = tf.argmax(self.y, -1)\n",
        "\n",
        "        loss = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.output, labels=self.y)\n",
        "        loss = loss * self.mask\n",
        "\n",
        "        self.loss = tf.reduce_sum(loss) / tf.reduce_sum(self.mask)\n",
        "\n",
        "    def _initialize_optimizer(self):\n",
        "        train_vars = tf.trainable_variables()\n",
        "        reg_loss = []\n",
        "        total_parameters = 0\n",
        "        for train_var in train_vars:\n",
        "            # print(train_var.name)\n",
        "            reg_loss.append(tf.nn.l2_loss(train_var))\n",
        "\n",
        "            shape = train_var.get_shape()\n",
        "            variable_parameters = 1\n",
        "            for dim in shape:\n",
        "                variable_parameters *= dim.value\n",
        "            total_parameters += variable_parameters\n",
        "        # print(total_parameters)\n",
        "        print('Trainable parameters:', total_parameters)\n",
        "\n",
        "        self.loss = self.loss + 0.00001 * tf.reduce_mean(reg_loss)\n",
        "        self.global_step = tf.get_variable(shape=[], initializer=tf.constant_initializer(0), dtype=tf.int32,\n",
        "                                           name='global_step')\n",
        "        self._optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.9, beta2=0.999)\n",
        "        # self._optimizer = tf.train.AdadeltaOptimizer(learning_rate=1.0, rho=0.95, epsilon=1e-08)\n",
        "\n",
        "        self.train_op = self._optimizer.minimize(self.loss, global_step=self.global_step)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WI01fUXOmb1f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed = 1234\n",
        "np.random.seed(seed)\n",
        "tf.set_random_seed(seed)\n",
        "unimodal_activations = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGBj3M8qmb1p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def multimodal(unimodal_activations, data, classes, attn_fusion=True, enable_attn_2=False, use_raw=True):\n",
        "    if use_raw:\n",
        "        if attn_fusion:\n",
        "            attn_fusion = False\n",
        "\n",
        "        train_data, test_data, audio_train, audio_test, text_train, text_test, video_train, video_test, train_label, test_label, seqlen_train, seqlen_test, train_mask, test_mask = get_raw_data(\n",
        "            data, classes)\n",
        "\n",
        "    else:\n",
        "        print(\"starting multimodal\")\n",
        "        # Fusion (appending) of features\n",
        "\n",
        "        text_train = unimodal_activations['text_train']\n",
        "        audio_train = unimodal_activations['audio_train']\n",
        "        video_train = unimodal_activations['video_train']\n",
        "\n",
        "        text_test = unimodal_activations['text_test']\n",
        "        audio_test = unimodal_activations['audio_test']\n",
        "        video_test = unimodal_activations['video_test']\n",
        "\n",
        "        train_mask = unimodal_activations['train_mask']\n",
        "        test_mask = unimodal_activations['test_mask']\n",
        "\n",
        "        print('train_mask', train_mask.shape)\n",
        "\n",
        "        train_label = unimodal_activations['train_label']\n",
        "        print('train_label', train_label.shape)\n",
        "        test_label = unimodal_activations['test_label']\n",
        "        print('test_label', test_label.shape)\n",
        "\n",
        "        # print(train_mask_bool)\n",
        "        seqlen_train = np.sum(train_mask, axis=-1)\n",
        "        print('seqlen_train', seqlen_train.shape)\n",
        "        seqlen_test = np.sum(test_mask, axis=-1)\n",
        "        print('seqlen_test', seqlen_test.shape)\n",
        "\n",
        "    a_dim = audio_train.shape[-1]\n",
        "    v_dim = video_train.shape[-1]\n",
        "    t_dim = text_train.shape[-1]\n",
        "    if attn_fusion:\n",
        "        print('With attention fusion')\n",
        "    allow_soft_placement = True\n",
        "    log_device_placement = False\n",
        "\n",
        "    # Multimodal model\n",
        "    session_conf = tf.ConfigProto(\n",
        "        # device_count={'GPU': gpu_count},\n",
        "        allow_soft_placement=allow_soft_placement,\n",
        "        log_device_placement=log_device_placement,\n",
        "        gpu_options=tf.GPUOptions(allow_growth=True))\n",
        "    gpu_device = 0\n",
        "    best_acc = 0\n",
        "    best_loss_accuracy = 0\n",
        "    best_loss = 10000000.0\n",
        "    best_epoch = 0\n",
        "    best_epoch_loss = 0\n",
        "    with tf.device('/device:GPU:%d' % gpu_device):\n",
        "        print('Using GPU - ', '/device:GPU:%d' % gpu_device)\n",
        "        with tf.Graph().as_default():\n",
        "            tf.set_random_seed(seed)\n",
        "            sess = tf.Session(config=session_conf)\n",
        "            with sess.as_default():\n",
        "                model = LSTM_Model(text_train.shape[1:], 0.0001, a_dim=a_dim, v_dim=v_dim, t_dim=t_dim,\n",
        "                                   emotions=classes, attn_fusion=attn_fusion,\n",
        "                                   unimodal=False, enable_attn_2=enable_attn_2,\n",
        "                                   seed=seed)\n",
        "                sess.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))\n",
        "\n",
        "                test_feed_dict = {\n",
        "                    model.t_input: text_test,\n",
        "                    model.a_input: audio_test,\n",
        "                    model.v_input: video_test,\n",
        "                    model.y: test_label,\n",
        "                    model.seq_len: seqlen_test,\n",
        "                    model.mask: test_mask,\n",
        "                    model.lstm_dropout: 0.0,\n",
        "                    model.lstm_inp_dropout: 0.0,\n",
        "                    model.dropout: 0.0,\n",
        "                    model.dropout_lstm_out: 0.0\n",
        "                }\n",
        "\n",
        "                # print('\\n\\nDataset: %s' % (data))\n",
        "                print(\"\\nEvaluation before training:\")\n",
        "                # Evaluation after epoch\n",
        "                step, loss, accuracy = sess.run(\n",
        "                    [model.global_step, model.loss, model.accuracy],\n",
        "                    test_feed_dict)\n",
        "                print(\"EVAL: epoch {}: step {}, loss {:g}, acc {:g}\".format(0, step, loss, accuracy))\n",
        "\n",
        "                for epoch in range(epochs):\n",
        "                    epoch += 1\n",
        "\n",
        "                    batches = batch_iter(list(\n",
        "                        zip(text_train, audio_train, video_train, train_mask, seqlen_train, train_label)),\n",
        "                        batch_size)\n",
        "\n",
        "                    # Training loop. For each batch...\n",
        "                    print('\\nTraining epoch {}'.format(epoch))\n",
        "                    l = []\n",
        "                    a = []\n",
        "                    for i, batch in tqdm(enumerate(batches)):\n",
        "                        b_text_train, b_audio_train, b_video_train, b_train_mask, b_seqlen_train, b_train_label = zip(\n",
        "                            *batch)\n",
        "                        # print('batch_hist_v', len(batch_utt_v))\n",
        "                        feed_dict = {\n",
        "                            model.t_input: b_text_train,\n",
        "                            model.a_input: b_audio_train,\n",
        "                            model.v_input: b_video_train,\n",
        "                            model.y: b_train_label,\n",
        "                            model.seq_len: b_seqlen_train,\n",
        "                            model.mask: b_train_mask,\n",
        "                            model.lstm_dropout: 0.4,\n",
        "                            model.lstm_inp_dropout: 0.0,\n",
        "                            model.dropout: 0.2,\n",
        "                            model.dropout_lstm_out: 0.2\n",
        "                        }\n",
        "\n",
        "                        _, step, loss, accuracy = sess.run(\n",
        "                            [model.train_op, model.global_step, model.loss, model.accuracy],\n",
        "                            feed_dict)\n",
        "                        l.append(loss)\n",
        "                        a.append(accuracy)\n",
        "\n",
        "                    print(\"\\t \\tEpoch {}:, loss {:g}, accuracy {:g}\".format(epoch, np.average(l), np.average(a)))\n",
        "                    # Evaluation after epoch\n",
        "                    step, loss, accuracy, preds, y, mask = sess.run(\n",
        "                        [model.global_step, model.loss, model.accuracy, model.preds, model.y, model.mask],\n",
        "                        test_feed_dict)\n",
        "                    f1 = f1_score(np.ndarray.flatten(tf.argmax(y, -1, output_type=tf.int32).eval()),\n",
        "                                  np.ndarray.flatten(tf.argmax(preds, -1, output_type=tf.int32).eval()),\n",
        "                                  sample_weight=np.ndarray.flatten(tf.cast(mask, tf.int32).eval()), average=\"weighted\")\n",
        "                    print(\"EVAL: After epoch {}: step {}, loss {:g}, acc {:g}, f1 {:g}\".format(epoch, step,\n",
        "                                                                                               loss / test_label.shape[\n",
        "                                                                                                   0],\n",
        "                                                                                               accuracy, f1))\n",
        "                    if accuracy > best_acc:\n",
        "                        best_epoch = epoch\n",
        "                        best_acc = accuracy\n",
        "                    if loss < best_loss:\n",
        "                        best_loss = loss\n",
        "                        best_loss_accuracy = accuracy\n",
        "                        best_epoch_loss = epoch\n",
        "\n",
        "                print(\n",
        "                    \"\\n\\nBest epoch: {}\\nBest test accuracy: {}\\nBest epoch loss: {}\\nBest test accuracy when loss is least: {}\".format(\n",
        "                        best_epoch, best_acc, best_epoch_loss, best_loss_accuracy))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2gj8eTwtmb2E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def unimodal(mode, data, classes):\n",
        "    print(('starting unimodal ', mode))\n",
        "\n",
        "    # with open('./mosei/text_glove_average.pickle', 'rb') as handle:\n",
        "    if data == 'mosei' or data == 'mosi':\n",
        "        with open('{1}_{2}way.pickle'.format(data, mode, classes), 'rb') as handle:\n",
        "            u = pickle._Unpickler(handle)\n",
        "            u.encoding = 'latin1'\n",
        "            # (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load()\n",
        "            if data == 'mosi':\n",
        "                (train_data, train_label, test_data, test_label, maxlen, train_length, test_length) = u.load()\n",
        "                train_label = train_label.astype('int')\n",
        "                test_label = test_label.astype('int')\n",
        "                train_label, test_label = createOneHot(train_label, test_label)\n",
        "            else:\n",
        "                raise NotImplementedError('Unknown dataset...')\n",
        "\n",
        "            train_label = train_label.astype('int')\n",
        "            test_label = test_label.astype('int')\n",
        "\n",
        "            train_mask = np.zeros((train_data.shape[0], train_data.shape[1]), dtype='float')\n",
        "            for i in range(len(train_length)):\n",
        "                train_mask[i, :train_length[i]] = 1.0\n",
        "\n",
        "            test_mask = np.zeros((test_data.shape[0], test_data.shape[1]), dtype='float')\n",
        "            for i in range(len(test_length)):\n",
        "                test_mask[i, :test_length[i]] = 1.0\n",
        "\n",
        "    attn_fusion = False\n",
        "\n",
        "    print('train_mask', train_mask.shape)\n",
        "\n",
        "    # print(train_mask_bool)\n",
        "    seqlen_train = np.sum(train_mask, axis=-1)\n",
        "    print('seqlen_train', seqlen_train.shape)\n",
        "    seqlen_test = np.sum(test_mask, axis=-1)\n",
        "    print('seqlen_test', seqlen_test.shape)\n",
        "\n",
        "    allow_soft_placement = True\n",
        "    log_device_placement = False\n",
        "\n",
        "    # Multimodal model\n",
        "    session_conf = tf.ConfigProto(\n",
        "        # device_count={'GPU': gpu_count},\n",
        "        allow_soft_placement=allow_soft_placement,\n",
        "        log_device_placement=log_device_placement,\n",
        "        gpu_options=tf.GPUOptions(allow_growth=True))\n",
        "    gpu_device = 0\n",
        "    best_acc = 0\n",
        "    best_epoch = 0\n",
        "    best_loss = 1000000.0\n",
        "    best_epoch_loss = 0\n",
        "    is_unimodal = True\n",
        "    with tf.device('/device:GPU:%d' % gpu_device):\n",
        "        print('Using GPU - ', '/device:GPU:%d' % gpu_device)\n",
        "        with tf.Graph().as_default():\n",
        "            tf.set_random_seed(seed)\n",
        "            sess = tf.Session(config=session_conf)\n",
        "            with sess.as_default():\n",
        "                model = LSTM_Model(train_data.shape[1:], 0.0001, a_dim=0, v_dim=0, t_dim=0, emotions=classes,\n",
        "                                   attn_fusion=attn_fusion, unimodal=is_unimodal, seed=seed)\n",
        "                sess.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))\n",
        "\n",
        "                test_feed_dict = {\n",
        "                    model.input: test_data,\n",
        "                    model.y: test_label,\n",
        "                    model.seq_len: seqlen_test,\n",
        "                    model.mask: test_mask,\n",
        "                    model.lstm_dropout: 0.0,\n",
        "                    model.lstm_inp_dropout: 0.0,\n",
        "                    model.dropout: 0.0,\n",
        "                    model.dropout_lstm_out: 0.0\n",
        "\n",
        "                }\n",
        "                train_feed_dict = {\n",
        "                    model.input: train_data,\n",
        "                    model.y: train_label,\n",
        "                    model.seq_len: seqlen_train,\n",
        "                    model.mask: train_mask,\n",
        "                    model.lstm_dropout: 0.0,\n",
        "                    model.lstm_inp_dropout: 0.0,\n",
        "                    model.dropout: 0.0,\n",
        "                    model.dropout_lstm_out: 0.0\n",
        "\n",
        "                }\n",
        "                # print('\\n\\nDataset: %s' % (data))\n",
        "                print(\"\\nEvaluation before training:\")\n",
        "                # Evaluation after epoch\n",
        "                step, loss, accuracy = sess.run(\n",
        "                    [model.global_step, model.loss, model.accuracy],\n",
        "                    test_feed_dict)\n",
        "                print(\"EVAL: epoch {}: step {}, loss {:g}, acc {:g}\".format(0, step, loss, accuracy))\n",
        "\n",
        "                for epoch in range(epochs):\n",
        "                    epoch += 1\n",
        "\n",
        "                    batches = batch_iter(list(\n",
        "                        zip(train_data, train_mask, seqlen_train, train_label)),\n",
        "                        batch_size)\n",
        "\n",
        "                    # Training loop. For each batch...\n",
        "                    print('\\nTraining epoch {}'.format(epoch))\n",
        "                    l = []\n",
        "                    a = []\n",
        "                    for i, batch in tqdm(enumerate(batches)):\n",
        "                        b_train_data, b_train_mask, b_seqlen_train, b_train_label = zip(\n",
        "                            *batch)\n",
        "                        feed_dict = {\n",
        "                            model.input: b_train_data,\n",
        "                            model.y: b_train_label,\n",
        "                            model.seq_len: b_seqlen_train,\n",
        "                            model.mask: b_train_mask,\n",
        "                            model.lstm_dropout: 0.4,\n",
        "                            model.lstm_inp_dropout: 0.0,\n",
        "                            model.dropout: 0.2,\n",
        "                            model.dropout_lstm_out: 0.2\n",
        "\n",
        "                        }\n",
        "\n",
        "                        _, step, loss, accuracy = sess.run(\n",
        "                            [model.train_op, model.global_step, model.loss, model.accuracy],\n",
        "                            feed_dict)\n",
        "                        l.append(loss)\n",
        "                        a.append(accuracy)\n",
        "\n",
        "                    print(\"\\t \\tEpoch {}:, loss {:g}, accuracy {:g}\".format(epoch, np.average(l), np.average(a)))\n",
        "                    # Evaluation after epoch\n",
        "                    step, loss, accuracy, test_activations = sess.run(\n",
        "                        [model.global_step, model.loss, model.accuracy, model.inter1],\n",
        "                        test_feed_dict)\n",
        "                    loss = loss / test_label.shape[0]\n",
        "                    print(\"EVAL: After epoch {}: step {}, loss {:g}, acc {:g}\".format(epoch, step, loss, accuracy))\n",
        "\n",
        "                    if accuracy > best_acc:\n",
        "                        best_epoch = epoch\n",
        "                        best_acc = accuracy\n",
        "\n",
        "                    if epoch == 30:\n",
        "                        step, loss, accuracy, train_activations = sess.run(\n",
        "                            [model.global_step, model.loss, model.accuracy, model.inter1],\n",
        "                            train_feed_dict)\n",
        "                        unimodal_activations[mode + '_train'] = train_activations\n",
        "                        unimodal_activations[mode + '_test'] = test_activations\n",
        "\n",
        "                        unimodal_activations['train_mask'] = train_mask\n",
        "                        unimodal_activations['test_mask'] = test_mask\n",
        "                        unimodal_activations['train_label'] = train_label\n",
        "                        unimodal_activations['test_label'] = test_label\n",
        "\n",
        "                    if loss < best_loss:\n",
        "                        best_epoch_loss = epoch\n",
        "                        best_loss = loss\n",
        "                        # step, loss, accuracy, train_activations = sess.run(\n",
        "                        # [model.global_step, model.loss, model.accuracy, model.inter1],\n",
        "                        # train_feed_dict)\n",
        "                        # unimodal_activations[mode + '_train'] = train_activations\n",
        "                        # unimodal_activations[mode + '_test'] = test_activations\n",
        "\n",
        "                        # unimodal_activations['train_mask'] = train_mask\n",
        "                        # unimodal_activations['test_mask'] = test_mask\n",
        "                        # unimodal_activations['train_label'] = train_label\n",
        "                        # unimodal_activations['test_label'] = test_label\n",
        "\n",
        "                print(\"\\n\\nBest epoch: {}\\nBest test accuracy: {}\".format(best_epoch, best_acc))\n",
        "                print(\"\\n\\nBest epoch: {}\\nBest test loss: {}\".format(best_epoch_loss, best_loss))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0I0bzj24mb2V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def str2bool(v):\n",
        "    if v.lower() in ('yes', 'true', 't', 'y', '1'):\n",
        "        return True\n",
        "    elif v.lower() in ('no', 'false', 'f', 'n', '0'):\n",
        "        return False\n",
        "    else:\n",
        "        raise argparse.ArgumentTypeError('Boolean value expected.')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQU2weyemb2k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 43081
        },
        "outputId": "16af12e9-0e01-4c2c-f00f-a3f6f60a3b13"
      },
      "source": [
        "if __name__ == \"__main__\":\n",
        "    argv = sys.argv[1:]\n",
        "    parser = argparse.ArgumentParser()\n",
        "    parser.add_argument(\"--unimodal\", type=str2bool, nargs='?', const=True, default=True)\n",
        "    parser.add_argument(\"--fusion\", type=str2bool, nargs='?', const=True, default=True)\n",
        "    parser.add_argument(\"--attention_2\", type=str2bool, nargs='?', const=True, default=True)\n",
        "    parser.add_argument(\"--use_raw\", type=str2bool, nargs='?', const=True, default=True)\n",
        "    parser.add_argument(\"--data\", type=str, default='mosi')\n",
        "    parser.add_argument(\"--classes\", type=str, default='2')\n",
        "    args, _ = parser.parse_known_args(argv)\n",
        "\n",
        "    print(args)\n",
        "\n",
        "    batch_size = 20\n",
        "    epochs = 100\n",
        "    emotions = args.classes\n",
        "    assert args.data in ['mosi', 'mosei', 'iemocap']\n",
        "\n",
        "    if args.unimodal:\n",
        "        print(\"Training unimodals first\")\n",
        "        modality = ['text', 'audio', 'video']\n",
        "        for mode in modality:\n",
        "            unimodal(mode, args.data, args.classes)\n",
        "\n",
        "        print(\"Saving unimodal activations\")\n",
        "        with open('unimodal_{0}_{1}way.pickle'.format(args.data, args.classes), 'wb') as handle:\n",
        "            pickle.dump(unimodal_activations, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "    if not args.use_raw:\n",
        "        with open('unimodal_{0}_{1}way.pickle'.format(args.data, args.classes), 'rb') as handle:\n",
        "            u = pickle._Unpickler(handle)\n",
        "            u.encoding = 'latin1'\n",
        "            unimodal_activations = u.load()\n",
        "\n",
        "    epochs = 100\n",
        "    multimodal(unimodal_activations, args.data, args.classes, args.fusion, args.attention_2, use_raw=args.use_raw)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Namespace(attention_2=True, classes='2', data='mosi', fusion=True, unimodal=True, use_raw=True)\n",
            "Training unimodals first\n",
            "('starting unimodal ', 'text')\n",
            "train_label shape (62, 63)\n",
            "test_label shape (31, 63)\n",
            "1\n",
            "1\n",
            "maxlen 1\n",
            "train [[[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "test [[[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "train_mask (62, 63)\n",
            "seqlen_train (62,)\n",
            "seqlen_test (31,)\n",
            "Using GPU -  /device:GPU:0\n",
            "Trainable parameters: 80602\n",
            "\n",
            "Evaluation before training:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: epoch 0: step 0, loss 0.725982, acc 0.331117\n",
            "\n",
            "Training epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.38it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 1:, loss 0.834817, accuracy 0.373755\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1it [00:00,  8.86it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 1: step 4, loss 0.0224069, acc 0.531915\n",
            "\n",
            "Training epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 2:, loss 0.753516, accuracy 0.442842\n",
            "EVAL: After epoch 2: step 8, loss 0.0215563, acc 0.644947\n",
            "\n",
            "Training epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.58it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 3:, loss 0.681082, accuracy 0.57506\n",
            "EVAL: After epoch 3: step 12, loss 0.0207846, acc 0.715426\n",
            "\n",
            "Training epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.92it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 4:, loss 0.654761, accuracy 0.592261\n",
            "EVAL: After epoch 4: step 16, loss 0.0201013, acc 0.746011\n",
            "\n",
            "Training epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.72it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 5:, loss 0.602947, accuracy 0.686567\n",
            "EVAL: After epoch 5: step 20, loss 0.019499, acc 0.755319\n",
            "\n",
            "Training epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 6:, loss 0.557411, accuracy 0.775464\n",
            "EVAL: After epoch 6: step 24, loss 0.0189631, acc 0.763298\n",
            "\n",
            "Training epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.43it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 7:, loss 0.518253, accuracy 0.81703\n",
            "EVAL: After epoch 7: step 28, loss 0.0184834, acc 0.765957\n",
            "\n",
            "Training epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 8:, loss 0.521075, accuracy 0.796257\n",
            "EVAL: After epoch 8: step 32, loss 0.0180516, acc 0.768617\n",
            "\n",
            "Training epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 9:, loss 0.471092, accuracy 0.833926\n",
            "EVAL: After epoch 9: step 36, loss 0.017669, acc 0.769947\n",
            "\n",
            "Training epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.16it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 10:, loss 0.455431, accuracy 0.85021\n",
            "EVAL: After epoch 10: step 40, loss 0.0173203, acc 0.772606\n",
            "\n",
            "Training epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.73it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 11:, loss 0.456117, accuracy 0.823347\n",
            "EVAL: After epoch 11: step 44, loss 0.0170114, acc 0.773936\n",
            "\n",
            "Training epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.89it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 12:, loss 0.425959, accuracy 0.824001\n",
            "EVAL: After epoch 12: step 48, loss 0.0167425, acc 0.777926\n",
            "\n",
            "Training epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.68it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 13:, loss 0.398226, accuracy 0.861646\n",
            "EVAL: After epoch 13: step 52, loss 0.0164994, acc 0.779255\n",
            "\n",
            "Training epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.52it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 14:, loss 0.394683, accuracy 0.852878\n",
            "EVAL: After epoch 14: step 56, loss 0.0162775, acc 0.779255\n",
            "\n",
            "Training epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.73it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 15:, loss 0.359024, accuracy 0.882331\n",
            "EVAL: After epoch 15: step 60, loss 0.0160767, acc 0.776596\n",
            "\n",
            "Training epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.84it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 16:, loss 0.370959, accuracy 0.879666\n",
            "EVAL: After epoch 16: step 64, loss 0.0158942, acc 0.780585\n",
            "\n",
            "Training epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 17:, loss 0.361558, accuracy 0.873591\n",
            "EVAL: After epoch 17: step 68, loss 0.0157253, acc 0.779255\n",
            "\n",
            "Training epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.78it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 18:, loss 0.349021, accuracy 0.868291\n",
            "EVAL: After epoch 18: step 72, loss 0.0155705, acc 0.779255\n",
            "\n",
            "Training epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 19:, loss 0.349926, accuracy 0.876104\n",
            "EVAL: After epoch 19: step 76, loss 0.0154286, acc 0.781915\n",
            "\n",
            "Training epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.34it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 20:, loss 0.328004, accuracy 0.886672\n",
            "EVAL: After epoch 20: step 80, loss 0.0152998, acc 0.783245\n",
            "\n",
            "Training epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 21:, loss 0.309764, accuracy 0.880203\n",
            "EVAL: After epoch 21: step 84, loss 0.0151846, acc 0.784574\n",
            "\n",
            "Training epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 22:, loss 0.276124, accuracy 0.908825\n",
            "EVAL: After epoch 22: step 88, loss 0.0150797, acc 0.783245\n",
            "\n",
            "Training epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 23:, loss 0.292561, accuracy 0.890995\n",
            "EVAL: After epoch 23: step 92, loss 0.0149813, acc 0.784574\n",
            "\n",
            "Training epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.77it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 24:, loss 0.264262, accuracy 0.912409\n",
            "EVAL: After epoch 24: step 96, loss 0.0148892, acc 0.788564\n",
            "\n",
            "Training epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.24it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 25:, loss 0.283387, accuracy 0.896429\n",
            "EVAL: After epoch 25: step 100, loss 0.0148056, acc 0.788564\n",
            "\n",
            "Training epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.93it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 26:, loss 0.28044, accuracy 0.894063\n",
            "EVAL: After epoch 26: step 104, loss 0.0147283, acc 0.789894\n",
            "\n",
            "Training epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.16it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 27:, loss 0.259, accuracy 0.92394\n",
            "EVAL: After epoch 27: step 108, loss 0.0146593, acc 0.791223\n",
            "\n",
            "Training epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.89it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 28:, loss 0.238362, accuracy 0.916797\n",
            "EVAL: After epoch 28: step 112, loss 0.0145943, acc 0.792553\n",
            "\n",
            "Training epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.56it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 29:, loss 0.234699, accuracy 0.923811\n",
            "EVAL: After epoch 29: step 116, loss 0.0145361, acc 0.792553\n",
            "\n",
            "Training epoch 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.18it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 30:, loss 0.232633, accuracy 0.921736\n",
            "EVAL: After epoch 30: step 120, loss 0.0144834, acc 0.797872\n",
            "\n",
            "Training epoch 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 31:, loss 0.241513, accuracy 0.923605\n",
            "EVAL: After epoch 31: step 124, loss 0.0144365, acc 0.799202\n",
            "\n",
            "Training epoch 32\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 32:, loss 0.225002, accuracy 0.918035\n",
            "EVAL: After epoch 32: step 128, loss 0.0143932, acc 0.799202\n",
            "\n",
            "Training epoch 33\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 33:, loss 0.223324, accuracy 0.916831\n",
            "EVAL: After epoch 33: step 132, loss 0.0143527, acc 0.800532\n",
            "\n",
            "Training epoch 34\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.89it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 34:, loss 0.203407, accuracy 0.940518\n",
            "EVAL: After epoch 34: step 136, loss 0.0143207, acc 0.804521\n",
            "\n",
            "Training epoch 35\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.94it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 35:, loss 0.198442, accuracy 0.930699\n",
            "EVAL: After epoch 35: step 140, loss 0.0143002, acc 0.807181\n",
            "\n",
            "Training epoch 36\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.05it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 36:, loss 0.183928, accuracy 0.951053\n",
            "EVAL: After epoch 36: step 144, loss 0.0142872, acc 0.804521\n",
            "\n",
            "Training epoch 37\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 37:, loss 0.177588, accuracy 0.935187\n",
            "EVAL: After epoch 37: step 148, loss 0.0142833, acc 0.808511\n",
            "\n",
            "Training epoch 38\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.08it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 38:, loss 0.181031, accuracy 0.950897\n",
            "EVAL: After epoch 38: step 152, loss 0.0142874, acc 0.808511\n",
            "\n",
            "Training epoch 39\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.57it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 39:, loss 0.192525, accuracy 0.948294\n",
            "EVAL: After epoch 39: step 156, loss 0.0142979, acc 0.805851\n",
            "\n",
            "Training epoch 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 40:, loss 0.169693, accuracy 0.942586\n",
            "EVAL: After epoch 40: step 160, loss 0.0143144, acc 0.804521\n",
            "\n",
            "Training epoch 41\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.81it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 41:, loss 0.156847, accuracy 0.950743\n",
            "EVAL: After epoch 41: step 164, loss 0.0143382, acc 0.801862\n",
            "\n",
            "Training epoch 42\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.86it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 42:, loss 0.180423, accuracy 0.931363\n",
            "EVAL: After epoch 42: step 168, loss 0.0143657, acc 0.799202\n",
            "\n",
            "Training epoch 43\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 43:, loss 0.14322, accuracy 0.964679\n",
            "EVAL: After epoch 43: step 172, loss 0.0144006, acc 0.800532\n",
            "\n",
            "Training epoch 44\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 44:, loss 0.144305, accuracy 0.966699\n",
            "EVAL: After epoch 44: step 176, loss 0.0144447, acc 0.801862\n",
            "\n",
            "Training epoch 45\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.64it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 45:, loss 0.134858, accuracy 0.968577\n",
            "EVAL: After epoch 45: step 180, loss 0.014501, acc 0.801862\n",
            "\n",
            "Training epoch 46\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 46:, loss 0.124956, accuracy 0.971353\n",
            "EVAL: After epoch 46: step 184, loss 0.014565, acc 0.799202\n",
            "\n",
            "Training epoch 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.20it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 47:, loss 0.117378, accuracy 0.973805\n",
            "EVAL: After epoch 47: step 188, loss 0.0146383, acc 0.796543\n",
            "\n",
            "Training epoch 48\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.88it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 48:, loss 0.120772, accuracy 0.967444\n",
            "EVAL: After epoch 48: step 192, loss 0.0147215, acc 0.795213\n",
            "\n",
            "Training epoch 49\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.19it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 49:, loss 0.122598, accuracy 0.969437\n",
            "EVAL: After epoch 49: step 196, loss 0.0148161, acc 0.795213\n",
            "\n",
            "Training epoch 50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 50:, loss 0.104515, accuracy 0.97701\n",
            "EVAL: After epoch 50: step 200, loss 0.0149131, acc 0.795213\n",
            "\n",
            "Training epoch 51\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.33it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 51:, loss 0.121666, accuracy 0.978046\n",
            "EVAL: After epoch 51: step 204, loss 0.015025, acc 0.795213\n",
            "\n",
            "Training epoch 52\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.55it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 52:, loss 0.0975594, accuracy 0.980075\n",
            "EVAL: After epoch 52: step 208, loss 0.0151547, acc 0.796543\n",
            "\n",
            "Training epoch 53\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.16it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 53:, loss 0.109158, accuracy 0.973523\n",
            "EVAL: After epoch 53: step 212, loss 0.0152863, acc 0.795213\n",
            "\n",
            "Training epoch 54\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.82it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 54:, loss 0.0944632, accuracy 0.971776\n",
            "EVAL: After epoch 54: step 216, loss 0.0154302, acc 0.793883\n",
            "\n",
            "Training epoch 55\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 55:, loss 0.0977132, accuracy 0.969991\n",
            "EVAL: After epoch 55: step 220, loss 0.0155657, acc 0.792553\n",
            "\n",
            "Training epoch 56\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.09it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 56:, loss 0.0910415, accuracy 0.980277\n",
            "EVAL: After epoch 56: step 224, loss 0.0156804, acc 0.792553\n",
            "\n",
            "Training epoch 57\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.33it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 57:, loss 0.0926884, accuracy 0.965458\n",
            "EVAL: After epoch 57: step 228, loss 0.0158051, acc 0.792553\n",
            "\n",
            "Training epoch 58\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 58:, loss 0.0784445, accuracy 0.986413\n",
            "EVAL: After epoch 58: step 232, loss 0.0159321, acc 0.789894\n",
            "\n",
            "Training epoch 59\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 59:, loss 0.0740775, accuracy 0.986135\n",
            "EVAL: After epoch 59: step 236, loss 0.0160794, acc 0.791223\n",
            "\n",
            "Training epoch 60\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 60:, loss 0.0835691, accuracy 0.979729\n",
            "EVAL: After epoch 60: step 240, loss 0.0162257, acc 0.792553\n",
            "\n",
            "Training epoch 61\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 61:, loss 0.0749694, accuracy 0.980879\n",
            "EVAL: After epoch 61: step 244, loss 0.0163561, acc 0.789894\n",
            "\n",
            "Training epoch 62\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.67it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 62:, loss 0.0691984, accuracy 0.986418\n",
            "EVAL: After epoch 62: step 248, loss 0.0164971, acc 0.789894\n",
            "\n",
            "Training epoch 63\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.42it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 63:, loss 0.0725868, accuracy 0.98768\n",
            "EVAL: After epoch 63: step 252, loss 0.0166408, acc 0.785904\n",
            "\n",
            "Training epoch 64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.62it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 64:, loss 0.0660382, accuracy 0.983253\n",
            "EVAL: After epoch 64: step 256, loss 0.0167821, acc 0.785904\n",
            "\n",
            "Training epoch 65\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.89it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 65:, loss 0.0697927, accuracy 0.985323\n",
            "EVAL: After epoch 65: step 260, loss 0.0169261, acc 0.788564\n",
            "\n",
            "Training epoch 66\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.40it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 66:, loss 0.09936, accuracy 0.975582\n",
            "EVAL: After epoch 66: step 264, loss 0.0170652, acc 0.789894\n",
            "\n",
            "Training epoch 67\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.19it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 67:, loss 0.0596415, accuracy 0.986068\n",
            "EVAL: After epoch 67: step 268, loss 0.0172083, acc 0.788564\n",
            "\n",
            "Training epoch 68\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 68:, loss 0.0623163, accuracy 0.985729\n",
            "EVAL: After epoch 68: step 272, loss 0.0173639, acc 0.785904\n",
            "\n",
            "Training epoch 69\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 69:, loss 0.0978555, accuracy 0.97512\n",
            "EVAL: After epoch 69: step 276, loss 0.0174922, acc 0.785904\n",
            "\n",
            "Training epoch 70\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.69it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 70:, loss 0.10667, accuracy 0.970717\n",
            "EVAL: After epoch 70: step 280, loss 0.0175619, acc 0.785904\n",
            "\n",
            "Training epoch 71\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.86it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 71:, loss 0.0589639, accuracy 0.987806\n",
            "EVAL: After epoch 71: step 284, loss 0.0176422, acc 0.787234\n",
            "\n",
            "Training epoch 72\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.51it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 72:, loss 0.0564814, accuracy 0.989192\n",
            "EVAL: After epoch 72: step 288, loss 0.0177649, acc 0.787234\n",
            "\n",
            "Training epoch 73\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.44it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 73:, loss 0.0764544, accuracy 0.976193\n",
            "EVAL: After epoch 73: step 292, loss 0.0178911, acc 0.787234\n",
            "\n",
            "Training epoch 74\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.32it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 74:, loss 0.0538711, accuracy 0.986422\n",
            "EVAL: After epoch 74: step 296, loss 0.017984, acc 0.788564\n",
            "\n",
            "Training epoch 75\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.53it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 75:, loss 0.0642483, accuracy 0.983881\n",
            "EVAL: After epoch 75: step 300, loss 0.0180914, acc 0.788564\n",
            "\n",
            "Training epoch 76\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.15it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 76:, loss 0.0569061, accuracy 0.981839\n",
            "EVAL: After epoch 76: step 304, loss 0.0182012, acc 0.787234\n",
            "\n",
            "Training epoch 77\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.61it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 77:, loss 0.0766997, accuracy 0.975848\n",
            "EVAL: After epoch 77: step 308, loss 0.0183084, acc 0.785904\n",
            "\n",
            "Training epoch 78\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.95it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 78:, loss 0.0519438, accuracy 0.986385\n",
            "EVAL: After epoch 78: step 312, loss 0.0184184, acc 0.788564\n",
            "\n",
            "Training epoch 79\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.56it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 79:, loss 0.0530087, accuracy 0.987111\n",
            "EVAL: After epoch 79: step 316, loss 0.018538, acc 0.789894\n",
            "\n",
            "Training epoch 80\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.41it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 80:, loss 0.0582843, accuracy 0.980617\n",
            "EVAL: After epoch 80: step 320, loss 0.0186608, acc 0.789894\n",
            "\n",
            "Training epoch 81\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.31it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 81:, loss 0.0799872, accuracy 0.971523\n",
            "EVAL: After epoch 81: step 324, loss 0.0187697, acc 0.789894\n",
            "\n",
            "Training epoch 82\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.75it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 82:, loss 0.0666907, accuracy 0.98235\n",
            "EVAL: After epoch 82: step 328, loss 0.0188321, acc 0.788564\n",
            "\n",
            "Training epoch 83\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 83:, loss 0.0687381, accuracy 0.970259\n",
            "EVAL: After epoch 83: step 332, loss 0.0188399, acc 0.784574\n",
            "\n",
            "Training epoch 84\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.23it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 84:, loss 0.108581, accuracy 0.969685\n",
            "EVAL: After epoch 84: step 336, loss 0.0188458, acc 0.779255\n",
            "\n",
            "Training epoch 85\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.62it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 85:, loss 0.0720394, accuracy 0.972345\n",
            "EVAL: After epoch 85: step 340, loss 0.0188624, acc 0.777926\n",
            "\n",
            "Training epoch 86\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.92it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 86:, loss 0.0618031, accuracy 0.980229\n",
            "EVAL: After epoch 86: step 344, loss 0.0189025, acc 0.775266\n",
            "\n",
            "Training epoch 87\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 87:, loss 0.0770093, accuracy 0.976157\n",
            "EVAL: After epoch 87: step 348, loss 0.0189648, acc 0.776596\n",
            "\n",
            "Training epoch 88\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.53it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 88:, loss 0.0483067, accuracy 0.987398\n",
            "EVAL: After epoch 88: step 352, loss 0.0190361, acc 0.779255\n",
            "\n",
            "Training epoch 89\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.49it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 89:, loss 0.0542918, accuracy 0.988803\n",
            "EVAL: After epoch 89: step 356, loss 0.0191292, acc 0.779255\n",
            "\n",
            "Training epoch 90\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 90:, loss 0.0465814, accuracy 0.986759\n",
            "EVAL: After epoch 90: step 360, loss 0.0192266, acc 0.779255\n",
            "\n",
            "Training epoch 91\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.69it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 91:, loss 0.0490346, accuracy 0.988328\n",
            "EVAL: After epoch 91: step 364, loss 0.0193197, acc 0.780585\n",
            "\n",
            "Training epoch 92\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.66it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 92:, loss 0.0476189, accuracy 0.988278\n",
            "EVAL: After epoch 92: step 368, loss 0.0194137, acc 0.781915\n",
            "\n",
            "Training epoch 93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.20it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 93:, loss 0.0505926, accuracy 0.984935\n",
            "EVAL: After epoch 93: step 372, loss 0.0194981, acc 0.781915\n",
            "\n",
            "Training epoch 94\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.75it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 94:, loss 0.100094, accuracy 0.964265\n",
            "EVAL: After epoch 94: step 376, loss 0.0195601, acc 0.781915\n",
            "\n",
            "Training epoch 95\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 95:, loss 0.0499455, accuracy 0.983485\n",
            "EVAL: After epoch 95: step 380, loss 0.0196007, acc 0.780585\n",
            "\n",
            "Training epoch 96\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.88it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 96:, loss 0.0481351, accuracy 0.987425\n",
            "EVAL: After epoch 96: step 384, loss 0.019664, acc 0.780585\n",
            "\n",
            "Training epoch 97\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.11it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 97:, loss 0.0501848, accuracy 0.986612\n",
            "EVAL: After epoch 97: step 388, loss 0.0197189, acc 0.780585\n",
            "\n",
            "Training epoch 98\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.43it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 98:, loss 0.0557888, accuracy 0.97894\n",
            "EVAL: After epoch 98: step 392, loss 0.0197921, acc 0.781915\n",
            "\n",
            "Training epoch 99\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 99:, loss 0.0636721, accuracy 0.976258\n",
            "EVAL: After epoch 99: step 396, loss 0.0198695, acc 0.781915\n",
            "\n",
            "Training epoch 100\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 100:, loss 0.0463463, accuracy 0.988031\n",
            "EVAL: After epoch 100: step 400, loss 0.0199071, acc 0.783245\n",
            "\n",
            "\n",
            "Best epoch: 37\n",
            "Best test accuracy: 0.8085106611251831\n",
            "\n",
            "\n",
            "Best epoch: 37\n",
            "Best test loss: 0.014283260991496424\n",
            "('starting unimodal ', 'audio')\n",
            "train_label shape (62, 63)\n",
            "test_label shape (31, 63)\n",
            "1\n",
            "1\n",
            "maxlen 1\n",
            "train [[[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "test [[[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "train_mask (62, 63)\n",
            "seqlen_train (62,)\n",
            "seqlen_test (31,)\n",
            "Using GPU -  /device:GPU:0\n",
            "Trainable parameters: 72502\n",
            "\n",
            "Evaluation before training:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: epoch 0: step 0, loss 0.736986, acc 0.621011\n",
            "\n",
            "Training epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  3.58it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 1:, loss 1.17537, accuracy 0.480823\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1it [00:00,  7.53it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 1: step 4, loss 0.0219277, acc 0.621011\n",
            "\n",
            "Training epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 2:, loss 1.01694, accuracy 0.496693\n",
            "EVAL: After epoch 2: step 8, loss 0.021676, acc 0.609043\n",
            "\n",
            "Training epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.01it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 3:, loss 0.894728, accuracy 0.516349\n",
            "EVAL: After epoch 3: step 12, loss 0.022764, acc 0.464096\n",
            "\n",
            "Training epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 4:, loss 0.906018, accuracy 0.48769\n",
            "EVAL: After epoch 4: step 16, loss 0.0247972, acc 0.386968\n",
            "\n",
            "Training epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.69it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 5:, loss 0.830925, accuracy 0.537904\n",
            "EVAL: After epoch 5: step 20, loss 0.0270169, acc 0.381649\n",
            "\n",
            "Training epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.43it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 6:, loss 0.919984, accuracy 0.484913\n",
            "EVAL: After epoch 6: step 24, loss 0.0287986, acc 0.378989\n",
            "\n",
            "Training epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.16it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 7:, loss 0.958511, accuracy 0.489204\n",
            "EVAL: After epoch 7: step 28, loss 0.0291932, acc 0.378989\n",
            "\n",
            "Training epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 8:, loss 0.8639, accuracy 0.523183\n",
            "EVAL: After epoch 8: step 32, loss 0.0284088, acc 0.378989\n",
            "\n",
            "Training epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.77it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 9:, loss 0.855099, accuracy 0.524494\n",
            "EVAL: After epoch 9: step 36, loss 0.0274139, acc 0.380319\n",
            "\n",
            "Training epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.88it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 10:, loss 0.829354, accuracy 0.519501\n",
            "EVAL: After epoch 10: step 40, loss 0.0266608, acc 0.381649\n",
            "\n",
            "Training epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.23it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 11:, loss 0.856901, accuracy 0.535154\n",
            "EVAL: After epoch 11: step 44, loss 0.0264421, acc 0.380319\n",
            "\n",
            "Training epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 12:, loss 0.898001, accuracy 0.498916\n",
            "EVAL: After epoch 12: step 48, loss 0.0262176, acc 0.380319\n",
            "\n",
            "Training epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 13:, loss 0.862998, accuracy 0.485207\n",
            "EVAL: After epoch 13: step 52, loss 0.0258877, acc 0.381649\n",
            "\n",
            "Training epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 14:, loss 0.838519, accuracy 0.51608\n",
            "EVAL: After epoch 14: step 56, loss 0.02555, acc 0.380319\n",
            "\n",
            "Training epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.12it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 15:, loss 0.835579, accuracy 0.495309\n",
            "EVAL: After epoch 15: step 60, loss 0.0254167, acc 0.378989\n",
            "\n",
            "Training epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.71it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 16:, loss 0.834773, accuracy 0.536933\n",
            "EVAL: After epoch 16: step 64, loss 0.0253153, acc 0.37766\n",
            "\n",
            "Training epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 17:, loss 0.877358, accuracy 0.511884\n",
            "EVAL: After epoch 17: step 68, loss 0.0252832, acc 0.378989\n",
            "\n",
            "Training epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.48it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 18:, loss 0.849537, accuracy 0.507003\n",
            "EVAL: After epoch 18: step 72, loss 0.0251879, acc 0.381649\n",
            "\n",
            "Training epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.14it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 19:, loss 0.894927, accuracy 0.471699\n",
            "EVAL: After epoch 19: step 76, loss 0.0255839, acc 0.381649\n",
            "\n",
            "Training epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.45it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 20:, loss 0.857744, accuracy 0.4977\n",
            "EVAL: After epoch 20: step 80, loss 0.0260981, acc 0.37766\n",
            "\n",
            "Training epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.76it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 21:, loss 0.862834, accuracy 0.496868\n",
            "EVAL: After epoch 21: step 84, loss 0.0260265, acc 0.378989\n",
            "\n",
            "Training epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.86it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 22:, loss 0.829947, accuracy 0.52135\n",
            "EVAL: After epoch 22: step 88, loss 0.0260506, acc 0.37633\n",
            "\n",
            "Training epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.51it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 23:, loss 0.798728, accuracy 0.551584\n",
            "EVAL: After epoch 23: step 92, loss 0.0260647, acc 0.37766\n",
            "\n",
            "Training epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 24:, loss 0.817978, accuracy 0.527369\n",
            "EVAL: After epoch 24: step 96, loss 0.0259768, acc 0.37766\n",
            "\n",
            "Training epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 25:, loss 0.833676, accuracy 0.512757\n",
            "EVAL: After epoch 25: step 100, loss 0.0258837, acc 0.37766\n",
            "\n",
            "Training epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 26:, loss 0.863457, accuracy 0.48592\n",
            "EVAL: After epoch 26: step 104, loss 0.0257065, acc 0.37766\n",
            "\n",
            "Training epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 27:, loss 0.791079, accuracy 0.557397\n",
            "EVAL: After epoch 27: step 108, loss 0.0254639, acc 0.378989\n",
            "\n",
            "Training epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 28:, loss 0.846593, accuracy 0.505462\n",
            "EVAL: After epoch 28: step 112, loss 0.0251364, acc 0.378989\n",
            "\n",
            "Training epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 29:, loss 0.798648, accuracy 0.514308\n",
            "EVAL: After epoch 29: step 116, loss 0.0249506, acc 0.378989\n",
            "\n",
            "Training epoch 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.18it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 30:, loss 0.793139, accuracy 0.525165\n",
            "EVAL: After epoch 30: step 120, loss 0.0247357, acc 0.378989\n",
            "\n",
            "Training epoch 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.95it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 31:, loss 0.835848, accuracy 0.493775\n",
            "EVAL: After epoch 31: step 124, loss 0.0246903, acc 0.37766\n",
            "\n",
            "Training epoch 32\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.93it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 32:, loss 0.798081, accuracy 0.511599\n",
            "EVAL: After epoch 32: step 128, loss 0.0246475, acc 0.378989\n",
            "\n",
            "Training epoch 33\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 33:, loss 0.798437, accuracy 0.505318\n",
            "EVAL: After epoch 33: step 132, loss 0.0244894, acc 0.378989\n",
            "\n",
            "Training epoch 34\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 34:, loss 0.868326, accuracy 0.494246\n",
            "EVAL: After epoch 34: step 136, loss 0.0241647, acc 0.37766\n",
            "\n",
            "Training epoch 35\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.15it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 35:, loss 0.841629, accuracy 0.494847\n",
            "EVAL: After epoch 35: step 140, loss 0.0242168, acc 0.378989\n",
            "\n",
            "Training epoch 36\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.81it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 36:, loss 0.806834, accuracy 0.473284\n",
            "EVAL: After epoch 36: step 144, loss 0.0245182, acc 0.37766\n",
            "\n",
            "Training epoch 37\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.27it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 37:, loss 0.799441, accuracy 0.523029\n",
            "EVAL: After epoch 37: step 148, loss 0.0247662, acc 0.37766\n",
            "\n",
            "Training epoch 38\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 38:, loss 0.793578, accuracy 0.505577\n",
            "EVAL: After epoch 38: step 152, loss 0.0250219, acc 0.378989\n",
            "\n",
            "Training epoch 39\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.67it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 39:, loss 0.815725, accuracy 0.502472\n",
            "EVAL: After epoch 39: step 156, loss 0.0253007, acc 0.378989\n",
            "\n",
            "Training epoch 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 40:, loss 0.823752, accuracy 0.496372\n",
            "EVAL: After epoch 40: step 160, loss 0.0251708, acc 0.378989\n",
            "\n",
            "Training epoch 41\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.85it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 41:, loss 0.77581, accuracy 0.538509\n",
            "EVAL: After epoch 41: step 164, loss 0.024968, acc 0.378989\n",
            "\n",
            "Training epoch 42\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.38it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 42:, loss 0.808496, accuracy 0.493966\n",
            "EVAL: After epoch 42: step 168, loss 0.0248038, acc 0.378989\n",
            "\n",
            "Training epoch 43\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 43:, loss 0.773893, accuracy 0.498346\n",
            "EVAL: After epoch 43: step 172, loss 0.0244723, acc 0.37766\n",
            "\n",
            "Training epoch 44\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.11it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 44:, loss 0.768583, accuracy 0.533969\n",
            "EVAL: After epoch 44: step 176, loss 0.0243572, acc 0.37766\n",
            "\n",
            "Training epoch 45\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.55it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 45:, loss 0.777963, accuracy 0.534025\n",
            "EVAL: After epoch 45: step 180, loss 0.024033, acc 0.381649\n",
            "\n",
            "Training epoch 46\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.56it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 46:, loss 0.793435, accuracy 0.527722\n",
            "EVAL: After epoch 46: step 184, loss 0.023828, acc 0.382979\n",
            "\n",
            "Training epoch 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 47:, loss 0.75988, accuracy 0.507427\n",
            "EVAL: After epoch 47: step 188, loss 0.0238558, acc 0.382979\n",
            "\n",
            "Training epoch 48\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.26it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 48:, loss 0.795032, accuracy 0.493074\n",
            "EVAL: After epoch 48: step 192, loss 0.0238884, acc 0.382979\n",
            "\n",
            "Training epoch 49\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.81it/s]\n",
            "1it [00:00,  9.48it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 49:, loss 0.826463, accuracy 0.450468\n",
            "EVAL: After epoch 49: step 196, loss 0.0240077, acc 0.380319\n",
            "\n",
            "Training epoch 50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.94it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 50:, loss 0.768366, accuracy 0.517515\n",
            "EVAL: After epoch 50: step 200, loss 0.0237434, acc 0.382979\n",
            "\n",
            "Training epoch 51\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 51:, loss 0.773639, accuracy 0.493707\n",
            "EVAL: After epoch 51: step 204, loss 0.0234975, acc 0.386968\n",
            "\n",
            "Training epoch 52\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.80it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 52:, loss 0.804575, accuracy 0.487388\n",
            "EVAL: After epoch 52: step 208, loss 0.0234979, acc 0.388298\n",
            "\n",
            "Training epoch 53\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 53:, loss 0.816164, accuracy 0.472596\n",
            "EVAL: After epoch 53: step 212, loss 0.0235194, acc 0.384309\n",
            "\n",
            "Training epoch 54\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.31it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 54:, loss 0.782537, accuracy 0.504269\n",
            "EVAL: After epoch 54: step 216, loss 0.0235705, acc 0.381649\n",
            "\n",
            "Training epoch 55\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.81it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 55:, loss 0.782928, accuracy 0.49557\n",
            "EVAL: After epoch 55: step 220, loss 0.0236124, acc 0.381649\n",
            "\n",
            "Training epoch 56\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 56:, loss 0.776102, accuracy 0.519006\n",
            "EVAL: After epoch 56: step 224, loss 0.0238988, acc 0.378989\n",
            "\n",
            "Training epoch 57\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.73it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 57:, loss 0.743458, accuracy 0.521354\n",
            "EVAL: After epoch 57: step 228, loss 0.0241109, acc 0.37766\n",
            "\n",
            "Training epoch 58\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.60it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 58:, loss 0.797121, accuracy 0.477451\n",
            "EVAL: After epoch 58: step 232, loss 0.0241578, acc 0.378989\n",
            "\n",
            "Training epoch 59\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.84it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 59:, loss 0.759337, accuracy 0.538032\n",
            "EVAL: After epoch 59: step 236, loss 0.0241786, acc 0.378989\n",
            "\n",
            "Training epoch 60\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 60:, loss 0.757346, accuracy 0.490318\n",
            "EVAL: After epoch 60: step 240, loss 0.0240653, acc 0.37766\n",
            "\n",
            "Training epoch 61\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.72it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 61:, loss 0.742132, accuracy 0.565202\n",
            "EVAL: After epoch 61: step 244, loss 0.0240502, acc 0.37766\n",
            "\n",
            "Training epoch 62\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.94it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 62:, loss 0.762044, accuracy 0.50957\n",
            "EVAL: After epoch 62: step 248, loss 0.0239121, acc 0.37766\n",
            "\n",
            "Training epoch 63\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 63:, loss 0.785287, accuracy 0.479949\n",
            "EVAL: After epoch 63: step 252, loss 0.0238571, acc 0.37766\n",
            "\n",
            "Training epoch 64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 64:, loss 0.775808, accuracy 0.50785\n",
            "EVAL: After epoch 64: step 256, loss 0.0238874, acc 0.378989\n",
            "\n",
            "Training epoch 65\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.93it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 65:, loss 0.774328, accuracy 0.498323\n",
            "EVAL: After epoch 65: step 260, loss 0.0241381, acc 0.378989\n",
            "\n",
            "Training epoch 66\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.34it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 66:, loss 0.784633, accuracy 0.47139\n",
            "EVAL: After epoch 66: step 264, loss 0.0244006, acc 0.378989\n",
            "\n",
            "Training epoch 67\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.72it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 67:, loss 0.767428, accuracy 0.517149\n",
            "EVAL: After epoch 67: step 268, loss 0.0247203, acc 0.378989\n",
            "\n",
            "Training epoch 68\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.82it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 68:, loss 0.76518, accuracy 0.487753\n",
            "EVAL: After epoch 68: step 272, loss 0.0248055, acc 0.378989\n",
            "\n",
            "Training epoch 69\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 69:, loss 0.753976, accuracy 0.512359\n",
            "EVAL: After epoch 69: step 276, loss 0.0246439, acc 0.378989\n",
            "\n",
            "Training epoch 70\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 70:, loss 0.746127, accuracy 0.517172\n",
            "EVAL: After epoch 70: step 280, loss 0.0245113, acc 0.378989\n",
            "\n",
            "Training epoch 71\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 71:, loss 0.759591, accuracy 0.49955\n",
            "EVAL: After epoch 71: step 284, loss 0.0245733, acc 0.378989\n",
            "\n",
            "Training epoch 72\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 72:, loss 0.7555, accuracy 0.503028\n",
            "EVAL: After epoch 72: step 288, loss 0.024632, acc 0.378989\n",
            "\n",
            "Training epoch 73\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.67it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 73:, loss 0.763646, accuracy 0.514819\n",
            "EVAL: After epoch 73: step 292, loss 0.0245534, acc 0.378989\n",
            "\n",
            "Training epoch 74\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 74:, loss 0.768005, accuracy 0.502213\n",
            "EVAL: After epoch 74: step 296, loss 0.0243349, acc 0.378989\n",
            "\n",
            "Training epoch 75\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.83it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 75:, loss 0.753122, accuracy 0.488283\n",
            "EVAL: After epoch 75: step 300, loss 0.0241156, acc 0.378989\n",
            "\n",
            "Training epoch 76\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.80it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 76:, loss 0.75486, accuracy 0.495613\n",
            "EVAL: After epoch 76: step 304, loss 0.0238817, acc 0.378989\n",
            "\n",
            "Training epoch 77\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.11it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 77:, loss 0.757502, accuracy 0.52034\n",
            "EVAL: After epoch 77: step 308, loss 0.0237287, acc 0.378989\n",
            "\n",
            "Training epoch 78\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 78:, loss 0.763768, accuracy 0.484344\n",
            "EVAL: After epoch 78: step 312, loss 0.0238185, acc 0.378989\n",
            "\n",
            "Training epoch 79\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.12it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 79:, loss 0.746246, accuracy 0.501194\n",
            "EVAL: After epoch 79: step 316, loss 0.0239308, acc 0.378989\n",
            "\n",
            "Training epoch 80\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.29it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 80:, loss 0.765483, accuracy 0.490739\n",
            "EVAL: After epoch 80: step 320, loss 0.0239319, acc 0.378989\n",
            "\n",
            "Training epoch 81\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.99it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 81:, loss 0.786416, accuracy 0.474514\n",
            "EVAL: After epoch 81: step 324, loss 0.0239354, acc 0.378989\n",
            "\n",
            "Training epoch 82\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 82:, loss 0.755431, accuracy 0.494868\n",
            "EVAL: After epoch 82: step 328, loss 0.0239286, acc 0.378989\n",
            "\n",
            "Training epoch 83\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.45it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 83:, loss 0.757849, accuracy 0.51061\n",
            "EVAL: After epoch 83: step 332, loss 0.0239308, acc 0.378989\n",
            "\n",
            "Training epoch 84\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.30it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 84:, loss 0.755711, accuracy 0.494271\n",
            "EVAL: After epoch 84: step 336, loss 0.0238951, acc 0.378989\n",
            "\n",
            "Training epoch 85\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.14it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 85:, loss 0.723366, accuracy 0.532884\n",
            "EVAL: After epoch 85: step 340, loss 0.0239107, acc 0.378989\n",
            "\n",
            "Training epoch 86\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.41it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 86:, loss 0.732531, accuracy 0.532142\n",
            "EVAL: After epoch 86: step 344, loss 0.0239234, acc 0.378989\n",
            "\n",
            "Training epoch 87\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 87:, loss 0.753138, accuracy 0.546302\n",
            "EVAL: After epoch 87: step 348, loss 0.0240043, acc 0.378989\n",
            "\n",
            "Training epoch 88\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.91it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 88:, loss 0.762969, accuracy 0.492707\n",
            "EVAL: After epoch 88: step 352, loss 0.0240894, acc 0.378989\n",
            "\n",
            "Training epoch 89\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 89:, loss 0.722523, accuracy 0.534303\n",
            "EVAL: After epoch 89: step 356, loss 0.0241887, acc 0.378989\n",
            "\n",
            "Training epoch 90\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.65it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 90:, loss 0.743301, accuracy 0.501351\n",
            "EVAL: After epoch 90: step 360, loss 0.0242153, acc 0.378989\n",
            "\n",
            "Training epoch 91\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 91:, loss 0.762654, accuracy 0.475567\n",
            "EVAL: After epoch 91: step 364, loss 0.0241845, acc 0.378989\n",
            "\n",
            "Training epoch 92\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.93it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 92:, loss 0.757696, accuracy 0.483916\n",
            "EVAL: After epoch 92: step 368, loss 0.024087, acc 0.378989\n",
            "\n",
            "Training epoch 93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 93:, loss 0.732799, accuracy 0.522203\n",
            "EVAL: After epoch 93: step 372, loss 0.0241307, acc 0.378989\n",
            "\n",
            "Training epoch 94\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.51it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 94:, loss 0.728938, accuracy 0.562114\n",
            "EVAL: After epoch 94: step 376, loss 0.0242203, acc 0.378989\n",
            "\n",
            "Training epoch 95\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.84it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 95:, loss 0.726558, accuracy 0.511126\n",
            "EVAL: After epoch 95: step 380, loss 0.0241716, acc 0.378989\n",
            "\n",
            "Training epoch 96\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 96:, loss 0.74324, accuracy 0.504986\n",
            "EVAL: After epoch 96: step 384, loss 0.0239615, acc 0.378989\n",
            "\n",
            "Training epoch 97\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.41it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 97:, loss 0.755825, accuracy 0.484228\n",
            "EVAL: After epoch 97: step 388, loss 0.0237831, acc 0.378989\n",
            "\n",
            "Training epoch 98\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.94it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 98:, loss 0.74389, accuracy 0.497098\n",
            "EVAL: After epoch 98: step 392, loss 0.0236281, acc 0.378989\n",
            "\n",
            "Training epoch 99\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.05it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 99:, loss 0.725098, accuracy 0.524146\n",
            "EVAL: After epoch 99: step 396, loss 0.023565, acc 0.378989\n",
            "\n",
            "Training epoch 100\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.33it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 100:, loss 0.733945, accuracy 0.518502\n",
            "EVAL: After epoch 100: step 400, loss 0.023602, acc 0.378989\n",
            "\n",
            "\n",
            "Best epoch: 1\n",
            "Best test accuracy: 0.6210106611251831\n",
            "\n",
            "\n",
            "Best epoch: 2\n",
            "Best test loss: 0.021676036619370984\n",
            "('starting unimodal ', 'video')\n",
            "train_label shape (62, 63)\n",
            "test_label shape (31, 63)\n",
            "1\n",
            "1\n",
            "maxlen 1\n",
            "train [[[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "test [[[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "train_mask (62, 63)\n",
            "seqlen_train (62,)\n",
            "seqlen_test (31,)\n",
            "Using GPU -  /device:GPU:0\n",
            "Trainable parameters: 80602\n",
            "\n",
            "Evaluation before training:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: epoch 0: step 0, loss 0.729511, acc 0.609043\n",
            "\n",
            "Training epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.27it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 1:, loss 0.996643, accuracy 0.530516\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1it [00:00,  7.23it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 1: step 4, loss 0.0231506, acc 0.594415\n",
            "\n",
            "Training epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.37it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 2:, loss 1.05951, accuracy 0.466904\n",
            "EVAL: After epoch 2: step 8, loss 0.0232705, acc 0.551862\n",
            "\n",
            "Training epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.51it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 3:, loss 0.864116, accuracy 0.529143\n",
            "EVAL: After epoch 3: step 12, loss 0.0235708, acc 0.523936\n",
            "\n",
            "Training epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.97it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 4:, loss 0.934713, accuracy 0.488922\n",
            "EVAL: After epoch 4: step 16, loss 0.0239725, acc 0.49867\n",
            "\n",
            "Training epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.98it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 5:, loss 0.847019, accuracy 0.533777\n",
            "EVAL: After epoch 5: step 20, loss 0.0244097, acc 0.477394\n",
            "\n",
            "Training epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.61it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 6:, loss 0.856757, accuracy 0.535906\n",
            "EVAL: After epoch 6: step 24, loss 0.024748, acc 0.466755\n",
            "\n",
            "Training epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.30it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 7:, loss 0.794058, accuracy 0.546516\n",
            "EVAL: After epoch 7: step 28, loss 0.0250754, acc 0.464096\n",
            "\n",
            "Training epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.26it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 8:, loss 0.757458, accuracy 0.582987\n",
            "EVAL: After epoch 8: step 32, loss 0.0253088, acc 0.461436\n",
            "\n",
            "Training epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.45it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 9:, loss 0.842876, accuracy 0.529412\n",
            "EVAL: After epoch 9: step 36, loss 0.025445, acc 0.473404\n",
            "\n",
            "Training epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.37it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 10:, loss 0.706555, accuracy 0.623244\n",
            "EVAL: After epoch 10: step 40, loss 0.0252048, acc 0.477394\n",
            "\n",
            "Training epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.56it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 11:, loss 0.718143, accuracy 0.623427\n",
            "EVAL: After epoch 11: step 44, loss 0.0248296, acc 0.493351\n",
            "\n",
            "Training epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.34it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 12:, loss 0.683617, accuracy 0.6316\n",
            "EVAL: After epoch 12: step 48, loss 0.0245289, acc 0.507979\n",
            "\n",
            "Training epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 13:, loss 0.662003, accuracy 0.651666\n",
            "EVAL: After epoch 13: step 52, loss 0.0243277, acc 0.519947\n",
            "\n",
            "Training epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.25it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 14:, loss 0.628741, accuracy 0.664839\n",
            "EVAL: After epoch 14: step 56, loss 0.0240844, acc 0.517287\n",
            "\n",
            "Training epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.74it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 15:, loss 0.643509, accuracy 0.643361\n",
            "EVAL: After epoch 15: step 60, loss 0.0238464, acc 0.521277\n",
            "\n",
            "Training epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 16:, loss 0.644639, accuracy 0.66345\n",
            "EVAL: After epoch 16: step 64, loss 0.0237306, acc 0.525266\n",
            "\n",
            "Training epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.42it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 17:, loss 0.577463, accuracy 0.70199\n",
            "EVAL: After epoch 17: step 68, loss 0.0236342, acc 0.531915\n",
            "\n",
            "Training epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 18:, loss 0.612951, accuracy 0.7015\n",
            "EVAL: After epoch 18: step 72, loss 0.0236505, acc 0.538564\n",
            "\n",
            "Training epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 19:, loss 0.56421, accuracy 0.723547\n",
            "EVAL: After epoch 19: step 76, loss 0.0237101, acc 0.538564\n",
            "\n",
            "Training epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.91it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 20:, loss 0.590137, accuracy 0.688452\n",
            "EVAL: After epoch 20: step 80, loss 0.023792, acc 0.533245\n",
            "\n",
            "Training epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.55it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 21:, loss 0.541426, accuracy 0.723067\n",
            "EVAL: After epoch 21: step 84, loss 0.0238638, acc 0.538564\n",
            "\n",
            "Training epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.16it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 22:, loss 0.615638, accuracy 0.681994\n",
            "EVAL: After epoch 22: step 88, loss 0.0240682, acc 0.539894\n",
            "\n",
            "Training epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.46it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 23:, loss 0.532838, accuracy 0.72997\n",
            "EVAL: After epoch 23: step 92, loss 0.0241686, acc 0.539894\n",
            "\n",
            "Training epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.99it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 24:, loss 0.509829, accuracy 0.758973\n",
            "EVAL: After epoch 24: step 96, loss 0.0243091, acc 0.538564\n",
            "\n",
            "Training epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.01it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 25:, loss 0.555141, accuracy 0.740167\n",
            "EVAL: After epoch 25: step 100, loss 0.0244846, acc 0.539894\n",
            "\n",
            "Training epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 26:, loss 0.543608, accuracy 0.73492\n",
            "EVAL: After epoch 26: step 104, loss 0.0245474, acc 0.541223\n",
            "\n",
            "Training epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.82it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 27:, loss 0.48027, accuracy 0.779657\n",
            "EVAL: After epoch 27: step 108, loss 0.0247076, acc 0.541223\n",
            "\n",
            "Training epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 28:, loss 0.491163, accuracy 0.780936\n",
            "EVAL: After epoch 28: step 112, loss 0.0248399, acc 0.542553\n",
            "\n",
            "Training epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.49it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 29:, loss 0.524278, accuracy 0.734511\n",
            "EVAL: After epoch 29: step 116, loss 0.0249784, acc 0.541223\n",
            "\n",
            "Training epoch 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.60it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 30:, loss 0.493546, accuracy 0.761324\n",
            "EVAL: After epoch 30: step 120, loss 0.0250545, acc 0.539894\n",
            "\n",
            "Training epoch 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.88it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 31:, loss 0.459724, accuracy 0.770372\n",
            "EVAL: After epoch 31: step 124, loss 0.025236, acc 0.543883\n",
            "\n",
            "Training epoch 32\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.74it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 32:, loss 0.504561, accuracy 0.766621\n",
            "EVAL: After epoch 32: step 128, loss 0.0254196, acc 0.546543\n",
            "\n",
            "Training epoch 33\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.67it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 33:, loss 0.504671, accuracy 0.748227\n",
            "EVAL: After epoch 33: step 132, loss 0.0254769, acc 0.542553\n",
            "\n",
            "Training epoch 34\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 34:, loss 0.468481, accuracy 0.767769\n",
            "EVAL: After epoch 34: step 136, loss 0.0255108, acc 0.543883\n",
            "\n",
            "Training epoch 35\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 35:, loss 0.433272, accuracy 0.791328\n",
            "EVAL: After epoch 35: step 140, loss 0.0255713, acc 0.547872\n",
            "\n",
            "Training epoch 36\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.74it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 36:, loss 0.422464, accuracy 0.819493\n",
            "EVAL: After epoch 36: step 144, loss 0.025651, acc 0.547872\n",
            "\n",
            "Training epoch 37\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 37:, loss 0.410563, accuracy 0.81546\n",
            "EVAL: After epoch 37: step 148, loss 0.0256354, acc 0.549202\n",
            "\n",
            "Training epoch 38\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.87it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 38:, loss 0.382659, accuracy 0.851364\n",
            "EVAL: After epoch 38: step 152, loss 0.025705, acc 0.549202\n",
            "\n",
            "Training epoch 39\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.33it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 39:, loss 0.400806, accuracy 0.816244\n",
            "EVAL: After epoch 39: step 156, loss 0.0258175, acc 0.549202\n",
            "\n",
            "Training epoch 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.62it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 40:, loss 0.428135, accuracy 0.803539\n",
            "EVAL: After epoch 40: step 160, loss 0.0259703, acc 0.553191\n",
            "\n",
            "Training epoch 41\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.21it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 41:, loss 0.463526, accuracy 0.788838\n",
            "EVAL: After epoch 41: step 164, loss 0.0261925, acc 0.554521\n",
            "\n",
            "Training epoch 42\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.73it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 42:, loss 0.42895, accuracy 0.815357\n",
            "EVAL: After epoch 42: step 168, loss 0.0264816, acc 0.555851\n",
            "\n",
            "Training epoch 43\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.48it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 43:, loss 0.411773, accuracy 0.835746\n",
            "EVAL: After epoch 43: step 172, loss 0.0267393, acc 0.553191\n",
            "\n",
            "Training epoch 44\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 44:, loss 0.439931, accuracy 0.808872\n",
            "EVAL: After epoch 44: step 176, loss 0.0269411, acc 0.549202\n",
            "\n",
            "Training epoch 45\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 45:, loss 0.417206, accuracy 0.822159\n",
            "EVAL: After epoch 45: step 180, loss 0.0268985, acc 0.551862\n",
            "\n",
            "Training epoch 46\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.92it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 46:, loss 0.413038, accuracy 0.802934\n",
            "EVAL: After epoch 46: step 184, loss 0.0267577, acc 0.555851\n",
            "\n",
            "Training epoch 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 47:, loss 0.39383, accuracy 0.8329\n",
            "EVAL: After epoch 47: step 188, loss 0.0266273, acc 0.56383\n",
            "\n",
            "Training epoch 48\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.39it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 48:, loss 0.408058, accuracy 0.815449\n",
            "EVAL: After epoch 48: step 192, loss 0.0267818, acc 0.56516\n",
            "\n",
            "Training epoch 49\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.04it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 49:, loss 0.349241, accuracy 0.859178\n",
            "EVAL: After epoch 49: step 196, loss 0.0270675, acc 0.55984\n",
            "\n",
            "Training epoch 50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.63it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 50:, loss 0.392882, accuracy 0.82074\n",
            "EVAL: After epoch 50: step 200, loss 0.0273068, acc 0.56117\n",
            "\n",
            "Training epoch 51\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.71it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 51:, loss 0.374271, accuracy 0.847614\n",
            "EVAL: After epoch 51: step 204, loss 0.0274742, acc 0.56117\n",
            "\n",
            "Training epoch 52\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.27it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 52:, loss 0.398816, accuracy 0.825538\n",
            "EVAL: After epoch 52: step 208, loss 0.0274805, acc 0.5625\n",
            "\n",
            "Training epoch 53\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.67it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 53:, loss 0.308206, accuracy 0.880889\n",
            "EVAL: After epoch 53: step 212, loss 0.0274356, acc 0.56117\n",
            "\n",
            "Training epoch 54\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 54:, loss 0.348871, accuracy 0.861812\n",
            "EVAL: After epoch 54: step 216, loss 0.0274817, acc 0.56117\n",
            "\n",
            "Training epoch 55\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 55:, loss 0.38057, accuracy 0.849432\n",
            "EVAL: After epoch 55: step 220, loss 0.0275382, acc 0.56117\n",
            "\n",
            "Training epoch 56\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.01it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 56:, loss 0.437483, accuracy 0.825238\n",
            "EVAL: After epoch 56: step 224, loss 0.0277032, acc 0.558511\n",
            "\n",
            "Training epoch 57\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 57:, loss 0.360483, accuracy 0.852635\n",
            "EVAL: After epoch 57: step 228, loss 0.0279918, acc 0.555851\n",
            "\n",
            "Training epoch 58\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.66it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 58:, loss 0.384432, accuracy 0.84117\n",
            "EVAL: After epoch 58: step 232, loss 0.0282977, acc 0.554521\n",
            "\n",
            "Training epoch 59\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.58it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 59:, loss 0.325757, accuracy 0.862702\n",
            "EVAL: After epoch 59: step 236, loss 0.0284003, acc 0.550532\n",
            "\n",
            "Training epoch 60\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.59it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 60:, loss 0.340149, accuracy 0.847947\n",
            "EVAL: After epoch 60: step 240, loss 0.0284325, acc 0.553191\n",
            "\n",
            "Training epoch 61\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.30it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 61:, loss 0.329362, accuracy 0.881559\n",
            "EVAL: After epoch 61: step 244, loss 0.0285701, acc 0.554521\n",
            "\n",
            "Training epoch 62\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 62:, loss 0.36175, accuracy 0.851312\n",
            "EVAL: After epoch 62: step 248, loss 0.0286945, acc 0.554521\n",
            "\n",
            "Training epoch 63\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.55it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 63:, loss 0.35514, accuracy 0.850507\n",
            "EVAL: After epoch 63: step 252, loss 0.0287202, acc 0.553191\n",
            "\n",
            "Training epoch 64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.34it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 64:, loss 0.310683, accuracy 0.878273\n",
            "EVAL: After epoch 64: step 256, loss 0.0288156, acc 0.554521\n",
            "\n",
            "Training epoch 65\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.69it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 65:, loss 0.327741, accuracy 0.878881\n",
            "EVAL: After epoch 65: step 260, loss 0.028999, acc 0.550532\n",
            "\n",
            "Training epoch 66\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.32it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 66:, loss 0.317398, accuracy 0.878069\n",
            "EVAL: After epoch 66: step 264, loss 0.0292293, acc 0.551862\n",
            "\n",
            "Training epoch 67\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.71it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 67:, loss 0.318052, accuracy 0.874591\n",
            "EVAL: After epoch 67: step 268, loss 0.0294533, acc 0.551862\n",
            "\n",
            "Training epoch 68\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.62it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 68:, loss 0.312494, accuracy 0.884494\n",
            "EVAL: After epoch 68: step 272, loss 0.0295768, acc 0.553191\n",
            "\n",
            "Training epoch 69\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.66it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 69:, loss 0.273458, accuracy 0.911704\n",
            "EVAL: After epoch 69: step 276, loss 0.0297748, acc 0.551862\n",
            "\n",
            "Training epoch 70\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.47it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 70:, loss 0.311475, accuracy 0.88529\n",
            "EVAL: After epoch 70: step 280, loss 0.0299303, acc 0.551862\n",
            "\n",
            "Training epoch 71\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 71:, loss 0.302498, accuracy 0.888268\n",
            "EVAL: After epoch 71: step 284, loss 0.0300272, acc 0.550532\n",
            "\n",
            "Training epoch 72\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.03it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 72:, loss 0.307251, accuracy 0.887077\n",
            "EVAL: After epoch 72: step 288, loss 0.0301553, acc 0.551862\n",
            "\n",
            "Training epoch 73\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 73:, loss 0.319711, accuracy 0.87826\n",
            "EVAL: After epoch 73: step 292, loss 0.0302866, acc 0.551862\n",
            "\n",
            "Training epoch 74\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 74:, loss 0.383545, accuracy 0.872179\n",
            "EVAL: After epoch 74: step 296, loss 0.03041, acc 0.550532\n",
            "\n",
            "Training epoch 75\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 75:, loss 0.264079, accuracy 0.906653\n",
            "EVAL: After epoch 75: step 300, loss 0.0305514, acc 0.550532\n",
            "\n",
            "Training epoch 76\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.58it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 76:, loss 0.265645, accuracy 0.897936\n",
            "EVAL: After epoch 76: step 304, loss 0.0306881, acc 0.550532\n",
            "\n",
            "Training epoch 77\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.17it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 77:, loss 0.328556, accuracy 0.884701\n",
            "EVAL: After epoch 77: step 308, loss 0.030823, acc 0.550532\n",
            "\n",
            "Training epoch 78\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.02it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 78:, loss 0.349991, accuracy 0.855185\n",
            "EVAL: After epoch 78: step 312, loss 0.0309794, acc 0.549202\n",
            "\n",
            "Training epoch 79\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.11it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 79:, loss 0.291988, accuracy 0.889509\n",
            "EVAL: After epoch 79: step 316, loss 0.0309905, acc 0.547872\n",
            "\n",
            "Training epoch 80\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 80:, loss 0.324786, accuracy 0.890133\n",
            "EVAL: After epoch 80: step 320, loss 0.0310246, acc 0.547872\n",
            "\n",
            "Training epoch 81\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.22it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 81:, loss 0.311356, accuracy 0.896203\n",
            "EVAL: After epoch 81: step 324, loss 0.0310469, acc 0.547872\n",
            "\n",
            "Training epoch 82\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.90it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 82:, loss 0.285468, accuracy 0.904785\n",
            "EVAL: After epoch 82: step 328, loss 0.0311473, acc 0.546543\n",
            "\n",
            "Training epoch 83\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.88it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 83:, loss 0.319343, accuracy 0.874847\n",
            "EVAL: After epoch 83: step 332, loss 0.0312384, acc 0.546543\n",
            "\n",
            "Training epoch 84\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.17it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 84:, loss 0.279162, accuracy 0.88839\n",
            "EVAL: After epoch 84: step 336, loss 0.0312698, acc 0.547872\n",
            "\n",
            "Training epoch 85\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.86it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 85:, loss 0.300468, accuracy 0.870595\n",
            "EVAL: After epoch 85: step 340, loss 0.0313406, acc 0.546543\n",
            "\n",
            "Training epoch 86\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.96it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 86:, loss 0.380734, accuracy 0.856781\n",
            "EVAL: After epoch 86: step 344, loss 0.0314458, acc 0.547872\n",
            "\n",
            "Training epoch 87\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.10it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 87:, loss 0.26405, accuracy 0.898515\n",
            "EVAL: After epoch 87: step 348, loss 0.0315623, acc 0.547872\n",
            "\n",
            "Training epoch 88\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.28it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 88:, loss 0.304126, accuracy 0.888525\n",
            "EVAL: After epoch 88: step 352, loss 0.0316854, acc 0.547872\n",
            "\n",
            "Training epoch 89\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.19it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 89:, loss 0.267815, accuracy 0.906145\n",
            "EVAL: After epoch 89: step 356, loss 0.0317859, acc 0.549202\n",
            "\n",
            "Training epoch 90\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.40it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 90:, loss 0.331009, accuracy 0.892368\n",
            "EVAL: After epoch 90: step 360, loss 0.0319075, acc 0.550532\n",
            "\n",
            "Training epoch 91\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.81it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 91:, loss 0.271766, accuracy 0.903297\n",
            "EVAL: After epoch 91: step 364, loss 0.0320855, acc 0.549202\n",
            "\n",
            "Training epoch 92\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.50it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 92:, loss 0.266205, accuracy 0.906467\n",
            "EVAL: After epoch 92: step 368, loss 0.0323394, acc 0.543883\n",
            "\n",
            "Training epoch 93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.15it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 93:, loss 0.266794, accuracy 0.900994\n",
            "EVAL: After epoch 93: step 372, loss 0.0325661, acc 0.542553\n",
            "\n",
            "Training epoch 94\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.62it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 94:, loss 0.291292, accuracy 0.905698\n",
            "EVAL: After epoch 94: step 376, loss 0.0327114, acc 0.542553\n",
            "\n",
            "Training epoch 95\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.06it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 95:, loss 0.251875, accuracy 0.911911\n",
            "EVAL: After epoch 95: step 380, loss 0.0327454, acc 0.543883\n",
            "\n",
            "Training epoch 96\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  6.99it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 96:, loss 0.259347, accuracy 0.913128\n",
            "EVAL: After epoch 96: step 384, loss 0.0328937, acc 0.545213\n",
            "\n",
            "Training epoch 97\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  8.60it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 97:, loss 0.259902, accuracy 0.91254\n",
            "EVAL: After epoch 97: step 388, loss 0.0330514, acc 0.545213\n",
            "\n",
            "Training epoch 98\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.54it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 98:, loss 0.244931, accuracy 0.913615\n",
            "EVAL: After epoch 98: step 392, loss 0.0331738, acc 0.543883\n",
            "\n",
            "Training epoch 99\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.36it/s]\n",
            "0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 99:, loss 0.234324, accuracy 0.917825\n",
            "EVAL: After epoch 99: step 396, loss 0.0333154, acc 0.542553\n",
            "\n",
            "Training epoch 100\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  7.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 100:, loss 0.382291, accuracy 0.840723\n",
            "EVAL: After epoch 100: step 400, loss 0.0334235, acc 0.541223\n",
            "\n",
            "\n",
            "Best epoch: 1\n",
            "Best test accuracy: 0.5944148898124695\n",
            "\n",
            "\n",
            "Best epoch: 1\n",
            "Best test loss: 0.02315056131732079\n",
            "Saving unimodal activations\n",
            "DATA [[[ 0.        0.        6.45163  ...  0.        0.        0.      ]\n",
            "  [ 0.       11.011     0.       ...  0.        1.54766   0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]\n",
            "\n",
            " [[ 0.        9.28649   0.       ...  0.        8.70788   0.      ]\n",
            "  [ 0.        0.        0.       ...  0.       81.1775    0.      ]\n",
            "  [ 0.        0.        0.       ...  0.       12.2632    0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]\n",
            "\n",
            " [[ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.806679  0.       ...  0.       21.2304    0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]\n",
            "\n",
            " [[ 0.        4.2682    0.       ...  0.        1.54244   0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        7.58169   0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]\n",
            "\n",
            " [[ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        2.27633   0.       ...  0.        4.73936   0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  ...\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]\n",
            "  [ 0.        0.        0.       ...  0.        0.        0.      ]]]\n",
            "shape (62, 63, 100)\n",
            "(62, 63, 273)\n",
            "(31, 63, 273)\n",
            "train_label shape (62, 63)\n",
            "test_label shape (31, 63)\n",
            "1\n",
            "1\n",
            "maxlen 1\n",
            "train [[[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "test [[[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [0. 1.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]]\n",
            "\n",
            " [[1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]\n",
            "\n",
            " [[0. 1.]\n",
            "  [0. 1.]\n",
            "  [0. 1.]\n",
            "  ...\n",
            "  [1. 0.]\n",
            "  [1. 0.]\n",
            "  [1. 0.]]]\n",
            "train_mask (62, 63)\n",
            "Using GPU -  /device:GPU:0\n",
            "Trainable parameters: 273402\n",
            "\n",
            "Evaluation before training:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: epoch 0: step 0, loss 0.678, acc 0.621011\n",
            "\n",
            "Training epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:05,  1.81s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 1:, loss 0.735309, accuracy 0.440596\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 1: step 4, loss 0.0221227, acc 0.555851, f1 0.45683\n",
            "\n",
            "Training epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 2:, loss 0.705846, accuracy 0.469653\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 2: step 8, loss 0.0228301, acc 0.37234, f1 0.268343\n",
            "\n",
            "Training epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 3:, loss 0.688478, accuracy 0.541569\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 3: step 12, loss 0.0233672, acc 0.378989, f1 0.208316\n",
            "\n",
            "Training epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 4:, loss 0.706494, accuracy 0.504704\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 4: step 16, loss 0.0233999, acc 0.378989, f1 0.208316\n",
            "\n",
            "Training epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.70it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 5:, loss 0.701639, accuracy 0.506518\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 5: step 20, loss 0.022508, acc 0.432181, f1 0.372932\n",
            "\n",
            "Training epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 6:, loss 0.690975, accuracy 0.527606\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 6: step 24, loss 0.0217632, acc 0.648936, f1 0.572065\n",
            "\n",
            "Training epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.67it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 7:, loss 0.666967, accuracy 0.616526\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 7: step 28, loss 0.0215651, acc 0.644947, f1 0.565076\n",
            "\n",
            "Training epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.70it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 8:, loss 0.687848, accuracy 0.535396\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 8: step 32, loss 0.0214443, acc 0.644947, f1 0.565076\n",
            "\n",
            "Training epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.41it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 9:, loss 0.673956, accuracy 0.580622\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 9: step 36, loss 0.0215739, acc 0.644947, f1 0.59533\n",
            "\n",
            "Training epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.66it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 10:, loss 0.674838, accuracy 0.596059\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 10: step 40, loss 0.0220207, acc 0.582447, f1 0.587998\n",
            "\n",
            "Training epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 11:, loss 0.669478, accuracy 0.587412\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 11: step 44, loss 0.0219881, acc 0.583777, f1 0.589436\n",
            "\n",
            "Training epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.64it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 12:, loss 0.657123, accuracy 0.618167\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 12: step 48, loss 0.0217365, acc 0.577128, f1 0.580683\n",
            "\n",
            "Training epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.85it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 13:, loss 0.644985, accuracy 0.663895\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 13: step 52, loss 0.0213991, acc 0.630319, f1 0.604216\n",
            "\n",
            "Training epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 14:, loss 0.657999, accuracy 0.634728\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 14: step 56, loss 0.0211978, acc 0.644947, f1 0.580904\n",
            "\n",
            "Training epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.75it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 15:, loss 0.651991, accuracy 0.651727\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 15: step 60, loss 0.021132, acc 0.644947, f1 0.603909\n",
            "\n",
            "Training epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 16:, loss 0.623671, accuracy 0.693855\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 16: step 64, loss 0.0212309, acc 0.614362, f1 0.60928\n",
            "\n",
            "Training epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 17:, loss 0.651779, accuracy 0.628362\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 17: step 68, loss 0.0214752, acc 0.621011, f1 0.622561\n",
            "\n",
            "Training epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.67it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 18:, loss 0.632694, accuracy 0.668661\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 18: step 72, loss 0.0213033, acc 0.619681, f1 0.617392\n",
            "\n",
            "Training epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 19:, loss 0.601569, accuracy 0.710871\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 19: step 76, loss 0.0210845, acc 0.618351, f1 0.610345\n",
            "\n",
            "Training epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 20:, loss 0.631315, accuracy 0.643039\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 20: step 80, loss 0.0211683, acc 0.62633, f1 0.62263\n",
            "\n",
            "Training epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 21:, loss 0.603192, accuracy 0.69663\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 21: step 84, loss 0.0212543, acc 0.631649, f1 0.631522\n",
            "\n",
            "Training epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.55it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 22:, loss 0.607407, accuracy 0.690726\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 22: step 88, loss 0.0209863, acc 0.630319, f1 0.62357\n",
            "\n",
            "Training epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 23:, loss 0.576316, accuracy 0.707921\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 23: step 92, loss 0.0206155, acc 0.635638, f1 0.617491\n",
            "\n",
            "Training epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.54it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 24:, loss 0.587514, accuracy 0.710158\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 24: step 96, loss 0.0205083, acc 0.631649, f1 0.609959\n",
            "\n",
            "Training epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 25:, loss 0.578518, accuracy 0.706271\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 25: step 100, loss 0.0205171, acc 0.632979, f1 0.614114\n",
            "\n",
            "Training epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 26:, loss 0.577635, accuracy 0.704377\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 26: step 104, loss 0.0206078, acc 0.642287, f1 0.632702\n",
            "\n",
            "Training epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  5.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 27:, loss 0.550423, accuracy 0.742228\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 27: step 108, loss 0.0205006, acc 0.636968, f1 0.623965\n",
            "\n",
            "Training epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.74it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 28:, loss 0.539773, accuracy 0.727583\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 28: step 112, loss 0.0203701, acc 0.647606, f1 0.622389\n",
            "\n",
            "Training epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 29:, loss 0.551169, accuracy 0.724777\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 29: step 116, loss 0.0203586, acc 0.656915, f1 0.625716\n",
            "\n",
            "Training epoch 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 30:, loss 0.534432, accuracy 0.742903\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 30: step 120, loss 0.0204529, acc 0.664894, f1 0.614635\n",
            "\n",
            "Training epoch 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 31:, loss 0.510599, accuracy 0.774412\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 31: step 124, loss 0.0203757, acc 0.654255, f1 0.631761\n",
            "\n",
            "Training epoch 32\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 32:, loss 0.518088, accuracy 0.776729\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 32: step 128, loss 0.0205607, acc 0.634309, f1 0.627041\n",
            "\n",
            "Training epoch 33\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.85it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 33:, loss 0.46405, accuracy 0.766063\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 33: step 132, loss 0.0204701, acc 0.660904, f1 0.627417\n",
            "\n",
            "Training epoch 34\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.73it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 34:, loss 0.503444, accuracy 0.765354\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 34: step 136, loss 0.0204496, acc 0.656915, f1 0.636417\n",
            "\n",
            "Training epoch 35\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 35:, loss 0.504563, accuracy 0.761702\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 35: step 140, loss 0.0207121, acc 0.639628, f1 0.632856\n",
            "\n",
            "Training epoch 36\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 36:, loss 0.462164, accuracy 0.77617\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 36: step 144, loss 0.020533, acc 0.660904, f1 0.632593\n",
            "\n",
            "Training epoch 37\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.49it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 37:, loss 0.495211, accuracy 0.766524\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 37: step 148, loss 0.0205544, acc 0.663564, f1 0.634058\n",
            "\n",
            "Training epoch 38\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 38:, loss 0.467739, accuracy 0.779674\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 38: step 152, loss 0.0206302, acc 0.635638, f1 0.629748\n",
            "\n",
            "Training epoch 39\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.72it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 39:, loss 0.489975, accuracy 0.740706\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 39: step 156, loss 0.0204163, acc 0.651596, f1 0.641179\n",
            "\n",
            "Training epoch 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.32it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 40:, loss 0.428446, accuracy 0.797716\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 40: step 160, loss 0.0205592, acc 0.670213, f1 0.642335\n",
            "\n",
            "Training epoch 41\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 41:, loss 0.467389, accuracy 0.806086\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 41: step 164, loss 0.0204889, acc 0.666223, f1 0.6477\n",
            "\n",
            "Training epoch 42\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.54it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 42:, loss 0.44606, accuracy 0.795089\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 42: step 168, loss 0.0206688, acc 0.644947, f1 0.642365\n",
            "\n",
            "Training epoch 43\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.80it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 43:, loss 0.429233, accuracy 0.801365\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 43: step 172, loss 0.0205312, acc 0.642287, f1 0.632702\n",
            "\n",
            "Training epoch 44\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  5.02it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 44:, loss 0.455633, accuracy 0.800516\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 44: step 176, loss 0.020975, acc 0.664894, f1 0.632939\n",
            "\n",
            "Training epoch 45\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.53it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 45:, loss 0.452297, accuracy 0.777514\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 45: step 180, loss 0.0213616, acc 0.664894, f1 0.632939\n",
            "\n",
            "Training epoch 46\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.61it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 46:, loss 0.421789, accuracy 0.816988\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 46: step 184, loss 0.020998, acc 0.650266, f1 0.643694\n",
            "\n",
            "Training epoch 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 47:, loss 0.469303, accuracy 0.781925\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 47: step 188, loss 0.0209206, acc 0.656915, f1 0.643432\n",
            "\n",
            "Training epoch 48\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 48:, loss 0.407039, accuracy 0.807006\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 48: step 192, loss 0.0208948, acc 0.668883, f1 0.648812\n",
            "\n",
            "Training epoch 49\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.82it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 49:, loss 0.400537, accuracy 0.814658\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 49: step 196, loss 0.020932, acc 0.664894, f1 0.643694\n",
            "\n",
            "Training epoch 50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.43it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 50:, loss 0.392423, accuracy 0.819789\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 50: step 200, loss 0.0210383, acc 0.654255, f1 0.638114\n",
            "\n",
            "Training epoch 51\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.87it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 51:, loss 0.371851, accuracy 0.851173\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 51: step 204, loss 0.0210987, acc 0.648936, f1 0.641766\n",
            "\n",
            "Training epoch 52\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 52:, loss 0.389812, accuracy 0.838078\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 52: step 208, loss 0.0215198, acc 0.662234, f1 0.639018\n",
            "\n",
            "Training epoch 53\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 53:, loss 0.37475, accuracy 0.853383\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 53: step 212, loss 0.0216757, acc 0.659574, f1 0.640397\n",
            "\n",
            "Training epoch 54\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.62it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 54:, loss 0.380057, accuracy 0.85174\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 54: step 216, loss 0.0215326, acc 0.644947, f1 0.637499\n",
            "\n",
            "Training epoch 55\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 55:, loss 0.347409, accuracy 0.849157\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 55: step 220, loss 0.0217073, acc 0.642287, f1 0.635566\n",
            "\n",
            "Training epoch 56\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.73it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 56:, loss 0.369899, accuracy 0.846261\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 56: step 224, loss 0.0221311, acc 0.655585, f1 0.63924\n",
            "\n",
            "Training epoch 57\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.42it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 57:, loss 0.363436, accuracy 0.854786\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 57: step 228, loss 0.022721, acc 0.666223, f1 0.637658\n",
            "\n",
            "Training epoch 58\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 58:, loss 0.357317, accuracy 0.852313\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 58: step 232, loss 0.021979, acc 0.656915, f1 0.652059\n",
            "\n",
            "Training epoch 59\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.44it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 59:, loss 0.369852, accuracy 0.838493\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 59: step 236, loss 0.0220318, acc 0.642287, f1 0.643106\n",
            "\n",
            "Training epoch 60\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 60:, loss 0.356802, accuracy 0.854385\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 60: step 240, loss 0.0225219, acc 0.655585, f1 0.635305\n",
            "\n",
            "Training epoch 61\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.74it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 61:, loss 0.315232, accuracy 0.864333\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 61: step 244, loss 0.0227551, acc 0.662234, f1 0.636426\n",
            "\n",
            "Training epoch 62\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 62:, loss 0.316482, accuracy 0.881601\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 62: step 248, loss 0.0221969, acc 0.658245, f1 0.649087\n",
            "\n",
            "Training epoch 63\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.81it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 63:, loss 0.343441, accuracy 0.865669\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 63: step 252, loss 0.022477, acc 0.647606, f1 0.631957\n",
            "\n",
            "Training epoch 64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.60it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 64:, loss 0.32289, accuracy 0.879537\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 64: step 256, loss 0.0223145, acc 0.654255, f1 0.642106\n",
            "\n",
            "Training epoch 65\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.87it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 65:, loss 0.313261, accuracy 0.861437\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 65: step 260, loss 0.0221653, acc 0.650266, f1 0.642137\n",
            "\n",
            "Training epoch 66\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.81it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 66:, loss 0.381267, accuracy 0.838986\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 66: step 264, loss 0.0223786, acc 0.654255, f1 0.64257\n",
            "\n",
            "Training epoch 67\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 67:, loss 0.315421, accuracy 0.885633\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 67: step 268, loss 0.0226658, acc 0.659574, f1 0.64262\n",
            "\n",
            "Training epoch 68\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 68:, loss 0.306433, accuracy 0.883085\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 68: step 272, loss 0.0225989, acc 0.654255, f1 0.641155\n",
            "\n",
            "Training epoch 69\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 69:, loss 0.308717, accuracy 0.89159\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 69: step 276, loss 0.0224562, acc 0.644947, f1 0.639024\n",
            "\n",
            "Training epoch 70\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.52it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 70:, loss 0.314618, accuracy 0.856164\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 70: step 280, loss 0.0229054, acc 0.656915, f1 0.644391\n",
            "\n",
            "Training epoch 71\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 71:, loss 0.353417, accuracy 0.847802\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 71: step 284, loss 0.0245499, acc 0.678191, f1 0.641401\n",
            "\n",
            "Training epoch 72\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.54it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 72:, loss 0.292903, accuracy 0.899799\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 72: step 288, loss 0.0235225, acc 0.652926, f1 0.63424\n",
            "\n",
            "Training epoch 73\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.64it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 73:, loss 0.295957, accuracy 0.883309\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 73: step 292, loss 0.0230163, acc 0.651596, f1 0.643697\n",
            "\n",
            "Training epoch 74\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 74:, loss 0.305294, accuracy 0.888042\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 74: step 296, loss 0.0229427, acc 0.648936, f1 0.647388\n",
            "\n",
            "Training epoch 75\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 75:, loss 0.298395, accuracy 0.898512\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 75: step 300, loss 0.0229641, acc 0.651596, f1 0.644861\n",
            "\n",
            "Training epoch 76\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  5.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 76:, loss 0.289573, accuracy 0.886837\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 76: step 304, loss 0.023978, acc 0.676862, f1 0.654349\n",
            "\n",
            "Training epoch 77\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 77:, loss 0.323969, accuracy 0.886343\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 77: step 308, loss 0.0237785, acc 0.670213, f1 0.651635\n",
            "\n",
            "Training epoch 78\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.79it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 78:, loss 0.289092, accuracy 0.896005\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 78: step 312, loss 0.0231604, acc 0.662234, f1 0.64896\n",
            "\n",
            "Training epoch 79\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.85it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 79:, loss 0.276531, accuracy 0.888307\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 79: step 316, loss 0.02313, acc 0.663564, f1 0.65105\n",
            "\n",
            "Training epoch 80\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.66it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 80:, loss 0.252492, accuracy 0.911782\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 80: step 320, loss 0.0235262, acc 0.670213, f1 0.654306\n",
            "\n",
            "Training epoch 81\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 81:, loss 0.29211, accuracy 0.884914\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 81: step 324, loss 0.0241123, acc 0.678191, f1 0.656072\n",
            "\n",
            "Training epoch 82\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.82it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 82:, loss 0.261562, accuracy 0.908086\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 82: step 328, loss 0.0233466, acc 0.675532, f1 0.665416\n",
            "\n",
            "Training epoch 83\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.62it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 83:, loss 0.256279, accuracy 0.896365\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 83: step 332, loss 0.0232408, acc 0.672872, f1 0.663907\n",
            "\n",
            "Training epoch 84\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 84:, loss 0.252325, accuracy 0.915082\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 84: step 336, loss 0.0242527, acc 0.672872, f1 0.649151\n",
            "\n",
            "Training epoch 85\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.64it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 85:, loss 0.281931, accuracy 0.901813\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 85: step 340, loss 0.0239321, acc 0.670213, f1 0.645015\n",
            "\n",
            "Training epoch 86\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  5.00it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 86:, loss 0.295072, accuracy 0.89889\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 86: step 344, loss 0.0234543, acc 0.668883, f1 0.652129\n",
            "\n",
            "Training epoch 87\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.51it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 87:, loss 0.295548, accuracy 0.889467\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 87: step 348, loss 0.0253603, acc 0.668883, f1 0.628919\n",
            "\n",
            "Training epoch 88\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.29it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 88:, loss 0.244651, accuracy 0.90117\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 88: step 352, loss 0.0235217, acc 0.682181, f1 0.664009\n",
            "\n",
            "Training epoch 89\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.52it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 89:, loss 0.242042, accuracy 0.914526\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 89: step 356, loss 0.0231878, acc 0.683511, f1 0.669217\n",
            "\n",
            "Training epoch 90\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.73it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 90:, loss 0.2174, accuracy 0.925725\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 90: step 360, loss 0.0236615, acc 0.680851, f1 0.662872\n",
            "\n",
            "Training epoch 91\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.82it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 91:, loss 0.231072, accuracy 0.924501\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 91: step 364, loss 0.0237884, acc 0.676862, f1 0.658386\n",
            "\n",
            "Training epoch 92\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.73it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 92:, loss 0.205073, accuracy 0.934346\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 92: step 368, loss 0.0245938, acc 0.678191, f1 0.649624\n",
            "\n",
            "Training epoch 93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 93:, loss 0.231819, accuracy 0.909837\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 93: step 372, loss 0.0247115, acc 0.678191, f1 0.649624\n",
            "\n",
            "Training epoch 94\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.81it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 94:, loss 0.224545, accuracy 0.929066\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 94: step 376, loss 0.0241134, acc 0.672872, f1 0.650993\n",
            "\n",
            "Training epoch 95\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.58it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 95:, loss 0.203678, accuracy 0.930385\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 95: step 380, loss 0.0237669, acc 0.664894, f1 0.64873\n",
            "\n",
            "Training epoch 96\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 96:, loss 0.292908, accuracy 0.89232\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 96: step 384, loss 0.0236007, acc 0.672872, f1 0.658589\n",
            "\n",
            "Training epoch 97\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.44it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 97:, loss 0.245388, accuracy 0.915499\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 97: step 388, loss 0.0237625, acc 0.69016, f1 0.670834\n",
            "\n",
            "Training epoch 98\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.77it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 98:, loss 0.217661, accuracy 0.929197\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 98: step 392, loss 0.0245995, acc 0.672872, f1 0.642407\n",
            "\n",
            "Training epoch 99\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.51it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 99:, loss 0.206769, accuracy 0.943054\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "EVAL: After epoch 99: step 396, loss 0.0242776, acc 0.674202, f1 0.644219\n",
            "\n",
            "Training epoch 100\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "4it [00:00,  4.82it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t \tEpoch 100:, loss 0.207859, accuracy 0.916177\n",
            "EVAL: After epoch 100: step 400, loss 0.0237695, acc 0.671543, f1 0.649875\n",
            "\n",
            "\n",
            "Best epoch: 97\n",
            "Best test accuracy: 0.6901595592498779\n",
            "Best epoch loss: 29\n",
            "Best test accuracy when loss is least: 0.6569148898124695\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3lNtr-emb2z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}