medical_text_generation_aitextgen.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/matthewchung74/thisbonedoesnotexist/blob/main/medical_text_generation_aitextgen.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5MfOTRtENRx_"
      },
      "outputs": [],
      "source": [
        "!pip install -q aitextgen"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VWxjrVuaNwNi"
      },
      "outputs": [],
      "source": [
        "from aitextgen.TokenDataset import TokenDataset\n",
        "from aitextgen.tokenizers import train_tokenizer\n",
        "from aitextgen.utils import GPT2ConfigCPU\n",
        "from aitextgen import aitextgen"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EXEB3it-ORWt",
        "outputId": "854cafbd-188e-4ad0-e79a-bde937dabf55"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "Note: using Google CoLab\n"
          ]
        }
      ],
      "source": [
        "try:\n",
        "    from google.colab import drive\n",
        "    drive.mount('/content/drive', force_remount=True)\n",
        "    COLAB = True\n",
        "    print(\"Note: using Google CoLab\")\n",
        "    %tensorflow_version 2.x\n",
        "except:\n",
        "    print(\"Note: not using Google CoLab\")\n",
        "    COLAB = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ENreTQQyi466",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8aeab2a7-f350-44dd-f3ec-3ad5e78c61d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mv: cannot stat 'noteevents.csv': No such file or directory\n"
          ]
        }
      ],
      "source": [
        "!mv noteevents.csv noteevents.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "670dd3e312b54ee28b501bc1a6195820",
            "142d62bf57454a5da3afc75990658102",
            "fb353421b0a94c5aa672c373713686f0",
            "0b73fa7ef1e649b288a81f2252bcc95a",
            "c7bdc880725d4bdc9619feb61b4ec438",
            "bf122ebd83a041a49d72e37d6c6f7024",
            "77f3b8bb16674754bca75d7373639613",
            "e190a83977bc4668aad9ab5b57fc4369",
            "efce1d9d081b4f92a40dc7d3a53fdf29",
            "1d57628dc7e945d78f7571b909c5e9f5",
            "643b0e21f6ac4e09afe1d2cbbbc08dec",
            "d1d58216696940c5ba8d199ca836be94",
            "df201a82f9e548d388a6037bea7dae28",
            "0e17aabbe6184e55921f84e979dcac43",
            "3525fdb2e81441c7b07cd064b6679b7b",
            "bbad5e3549ef4b6db1836e3531c07999",
            "78407d86c2f9431cbda7098877acb6b1",
            "267fc3ea8db74d4aaa797607c8010757",
            "3ff0bd22972346d0ad7f6ddb73d70d8b",
            "fb95c1d73f4e45c5b9146a407d3a0c76",
            "1d57e32bab614249a19307fddc4ea726",
            "6b07e6a87e9a4007ba4abb9c03812604"
          ]
        },
        "id": "XqozmJXDN0mt",
        "outputId": "60f005f9-9d71-4973-b671-c2f3a0cb0bb4"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2467666 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "670dd3e312b54ee28b501bc1a6195820"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "pytorch_model.bin already exists in /trained_model and will be overwritten!\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/connectors/callback_connector.py:152: LightningDeprecationWarning: Setting `Trainer(checkpoint_callback=False)` is deprecated in v1.5 and will be removed in v1.7. Please consider using `Trainer(enable_checkpointing=False)`.\n",
            "  f\"Setting `Trainer(checkpoint_callback={checkpoint_callback})` is deprecated in v1.5 and will \"\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/connectors/callback_connector.py:97: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=20)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.\n",
            "  f\"Setting `Trainer(progress_bar_refresh_rate={progress_bar_refresh_rate})` is deprecated in v1.5 and\"\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/connectors/callback_connector.py:172: LightningDeprecationWarning: Setting `Trainer(weights_summary=None)` is deprecated in v1.5 and will be removed in v1.7. Please set `Trainer(enable_model_summary=False)` instead.\n",
            "  \"Setting `Trainer(weights_summary=None)` is deprecated in v1.5 and will be removed\"\n",
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "IPU available: False, using: 0 IPUs\n",
            "HPU available: False, using: 0 HPUs\n",
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/configuration_validator.py:377: LightningDeprecationWarning: The `Callback.on_batch_end` hook was deprecated in v1.6 and will be removed in v1.8. Please use `Callback.on_train_batch_end` instead.\n",
            "  f\"The `Callback.{hook}` hook was deprecated in v1.6 and\"\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/50000 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d1d58216696940c5ba8d199ca836be94"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py:2282: LightningDeprecationWarning: `trainer.progress_bar_dict` is deprecated in v1.5 and will be removed in v1.7. Use `ProgressBarBase.get_metrics` instead.\n",
            "  \"`trainer.progress_bar_dict` is deprecated in v1.5 and will be removed in v1.7.\"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m5,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m5,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "\n",
            "     ______________________________________________________________________________\n",
            "     UNDERLYING MEDICAL CONDITION:\n",
            "      80 year old woman with ETT/porsia, status changes.\n",
            "     REASON FOR THIS EXAMINATION:\n",
            "      Please eval for h/ CVD.                                \n",
            "      \n",
            "     REASON FOR THIS EXAMINATION:\n",
            "\n",
            "==========\n",
            "\u001b[1m10,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m10,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            " to\n",
            "     aneurysm.\n",
            "     \n",
            "     There is mild difficultyroticulated attemporal just\n",
            "     size, swar size in the left lung.  No pneumothorax.\n",
            "     \n",
            "\n",
            "==========\n",
            "\u001b[1m15,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m15,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ".  Right, accordings.\n",
            "     \n",
            "     Planced telease due to pancreasing of the chest transferred to the patient's and bonchiefling of [**3305-2-21**].\n",
            "     \n",
            "\n",
            "==========\n",
            "\u001b[1m20,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m20,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ":\n",
            "holestomy tube, slightly decreased via\n",
            "dift.  Withoodynamically \"2 physiographics (1). \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Discharge Medical Date:\n",
            "\n",
            "==========\n",
            "\u001b[1m25,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m25,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "\n",
            "     demonstrating the acquumbrile.\n",
            "     \n",
            "     IMPRESSION:\n",
            "     1)  No evidence of pneumonia and multiple outpine demonstratous anestheopathy.\n",
            "     \n",
            "     ET tube in the left upper\n",
            "==========\n",
            "\u001b[1m30,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m30,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ":\n",
            "\n",
            "NEURO:  Tolerating, [**First Name8 (NamePattern2) **] [**D Number 13630**] & [**First Name8(MD) **] [**Last Name (NamePattern1) 377**], [**Name8 (PTT) **]\n",
            "==========\n",
            "\u001b[1m35,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m35,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            ".  This pused by Dr. [**First Name (STitle) **] [**Last Name (STitle) 1273**], [**Name (STitle) **] [**Last Name (STitle) **].\n",
            "     [**First Name (STitle) 252**]\n",
            " on: [**First Name8\n",
            "==========\n",
            "\u001b[1m40,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m40,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            "\n",
            "     change in the right hilar artery\n",
            "     is seen bilaterally.  The maxillary sigmoid system.  Nonog the right\n",
            "     pleural effusion, with several, spleen, unchanged in the liver\n",
            "==========\n",
            "\u001b[1m45,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m45,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            " status and sares.\n",
            "\",,\n",
            "\"2202\",\"12489\",\"300\",\"2/8/3321 02:31:00\",\"11/7/3051 02:16:00\",\"68\",\"N\",\"1\",\"Nursing/Other\",,\"\n",
            "\n",
            "==========\n",
            "\u001b[1m50,000 steps reached: saving model to /trained_model\u001b[0m\n",
            "\u001b[1m50,000 steps reached: generating sample texts.\u001b[0m\n",
            "==========\n",
            " in this\n",
            "     in the right lateral femoral artery.  The left and upper extravasation of the\n",
            "     jejunostomy, and into the right brachiograms.  The AVA x-ray,\n",
            "    \n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m from [**3357-9-8-6**], there is no further associated\n",
            "     of the right hepatumed pancreasing the side.  The heart and size is\n",
            "     advanced\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m.  The patient\n",
            "     intensive day.  Angiogram was utextubated for a Care Upon\n",
            "     were obtained from the table to the procedure well.\n",
            "     \n",
            "     FINDINGS:\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0ms.\n",
            "     \n",
            "     Bone windowel observing viewing the apical SVC.  The sheath was placed and\n",
            "     the guide wire was terminated in the baseron.  The\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m pancreatteral calcifications. The\n",
            "     gallbladders were discussed to procedure.  The patient was tided to the\n",
            "     verteably and Dr. [**Last Name (STitle\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m. [**MDoctor First Name **] was discussued with acidualsion to the days, the\n",
            "hematociumpressible, a wire was then removed with the patient has been\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m.\n",
            "     \n",
            "     FINDINGS: Doppler in the right superior brain, there is no significant change in the\n",
            "     coronalcifications.  This is stable in the left lower lobe, and\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m of the chest x-ray.  The cortogram was\n",
            "     patent and removed.  The patient had a questhesia of air iliac\n",
            "     and calcles in the left anterior\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m from the non-continued to\n",
            "      Please air.  Please breefusions for metastasis\n",
            "      Contrast: OPTIRAY Amt: 250\n",
            "     ______________________________________________________________________________\n",
            "     UNDERLYING MEDICAL\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m.\n",
            "     \n",
            "     FINDINGS:  Embolization of persistent spinally mass is present.  Small\n",
            "     vascularity, but may represent any of the\n",
            "splace disc sp\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m.\n",
            "\n",
            "Discharge Disposed.\n",
            "Dispo:\n",
            "Disp-physical Ee\n",
            "[**299-5-12-6**], MD [**285-30**] 3:14PM BLOOD\n"
          ]
        }
      ],
      "source": [
        "from aitextgen.TokenDataset import TokenDataset\n",
        "from aitextgen.tokenizers import train_tokenizer\n",
        "from aitextgen.utils import GPT2ConfigCPU\n",
        "from aitextgen import aitextgen\n",
        "\n",
        "file_name = \"noteevents.txt\"\n",
        "\n",
        "# Train a custom BPE Tokenizer on the downloaded text\n",
        "# This will save one file: `aitextgen.tokenizer.json`, which contains the\n",
        "# information needed to rebuild the tokenizer.\n",
        "train_tokenizer(file_name)\n",
        "tokenizer_file = \"aitextgen.tokenizer.json\"\n",
        "\n",
        "# GPT2ConfigCPU is a mini variant of GPT-2 optimized for CPU-training\n",
        "# e.g. the # of input tokens here is 64 vs. 1024 for base GPT-2.\n",
        "config = GPT2ConfigCPU()\n",
        "\n",
        "# Instantiate aitextgen using the created tokenizer and config\n",
        "ai = aitextgen(tokenizer_file=tokenizer_file, config=config)\n",
        "\n",
        "# You can build datasets for training by creating TokenDatasets,\n",
        "# which automatically processes the dataset with the appropriate size.\n",
        "data = TokenDataset(file_name, tokenizer_file=tokenizer_file, block_size=64)\n",
        "\n",
        "# Train the model! It will save pytorch_model.bin periodically and after completion to the `trained_model` folder.\n",
        "# On a 2020 8-core iMac, this took ~25 minutes to run.\n",
        "ai.train(data, batch_size=8, num_steps=50000, generate_every=5000, save_every=5000)\n",
        "\n",
        "# Generate text from it!\n",
        "ai.generate(10, prompt=\"The chest xray\", max_length=1000)\n",
        "\n",
        "# With your trained model, you can reload the model at any time by\n",
        "# providing the folder containing the pytorch_model.bin model weights + the config, and providing the tokenizer.\n",
        "ai2 = aitextgen(model_folder=\"trained_model\",\n",
        "                tokenizer_file=\"aitextgen.tokenizer.json\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ai.generate(3, prompt=\"The chest xray\", max_length=1000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OmBYm5hQ2eMq",
        "outputId": "3c2b5e02-ccf4-40ba-d064-354f06fc08d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mThe chest xray\u001b[0m.\n",
            "     \n",
            "     HISTORY:  On [**Initials (NamePatternt) **] [**Last Name (NamePattern4) **] [**Last Name (NamePattern4**], a 44**] [**Last Name (NamePattern1) 195**], [**Last Name\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m into the lung base.\n",
            "     \n",
            "     PROCESS:  The patient was transferratedly to the patient's Hip to be decided by\n",
            "reexprelative day.\n",
            "     \n",
            "     FINDINGS:\n",
            "==========\n",
            "\u001b[1mThe chest xray\u001b[0m from [**266-3-7**].\n",
            "     \n",
            "     FINDINGS:  The patient was placed supine on [**2632-4-6-4**].  The patient is acribed at the\n",
            "     utiliac sagitt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ai.generate(3, max_length=1000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TRtJPAiH2iJe",
        "outputId": "c2bf2203-c248-4c54-e63c-ca18d067205b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ".\n",
            "     and there was now signaline complainment of the marked stenosis and kidneys.  The\n",
            "     right middles of the right lower lobe, and greater than left, and left front\n",
            "==========\n",
            "and metastasis.  There is a small left retrocardiac opacity incidentally,\n",
            "     perihelates insupped with increased left base.  There is no free air.  No\n",
            "==========\n",
            "w/o hydropicatinary or portion of the iliac herniation.\n",
            "     REASON FOR THIS EXAMINATION:\n",
            "      S/P CABG, central line placement                                                                \n",
            "     ______________________________________________________________________________\n",
            "                                     FINAL REPORT\n",
            "     INDICATION: \n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "medical_text_generation_aitextgen.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMiILsdUlUC+DzARC/sBkkZ",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "670dd3e312b54ee28b501bc1a6195820": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_142d62bf57454a5da3afc75990658102",
              "IPY_MODEL_fb353421b0a94c5aa672c373713686f0",
              "IPY_MODEL_0b73fa7ef1e649b288a81f2252bcc95a"
            ],
            "layout": "IPY_MODEL_c7bdc880725d4bdc9619feb61b4ec438"
          }
        },
        "142d62bf57454a5da3afc75990658102": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf122ebd83a041a49d72e37d6c6f7024",
            "placeholder": "​",
            "style": "IPY_MODEL_77f3b8bb16674754bca75d7373639613",
            "value": "100%"
          }
        },
        "fb353421b0a94c5aa672c373713686f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e190a83977bc4668aad9ab5b57fc4369",
            "max": 2467666,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_efce1d9d081b4f92a40dc7d3a53fdf29",
            "value": 2467666
          }
        },
        "0b73fa7ef1e649b288a81f2252bcc95a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d57628dc7e945d78f7571b909c5e9f5",
            "placeholder": "​",
            "style": "IPY_MODEL_643b0e21f6ac4e09afe1d2cbbbc08dec",
            "value": " 2467666/2467666 [01:22&lt;00:00, 29740.48it/s]"
          }
        },
        "c7bdc880725d4bdc9619feb61b4ec438": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "100%"
          }
        },
        "bf122ebd83a041a49d72e37d6c6f7024": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77f3b8bb16674754bca75d7373639613": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e190a83977bc4668aad9ab5b57fc4369": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "efce1d9d081b4f92a40dc7d3a53fdf29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d57628dc7e945d78f7571b909c5e9f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "643b0e21f6ac4e09afe1d2cbbbc08dec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d1d58216696940c5ba8d199ca836be94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_df201a82f9e548d388a6037bea7dae28",
              "IPY_MODEL_0e17aabbe6184e55921f84e979dcac43",
              "IPY_MODEL_3525fdb2e81441c7b07cd064b6679b7b"
            ],
            "layout": "IPY_MODEL_bbad5e3549ef4b6db1836e3531c07999"
          }
        },
        "df201a82f9e548d388a6037bea7dae28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78407d86c2f9431cbda7098877acb6b1",
            "placeholder": "​",
            "style": "IPY_MODEL_267fc3ea8db74d4aaa797607c8010757",
            "value": "Loss: 2.450 — Avg: 2.504 — GPU Mem: 939 MB: 100%"
          }
        },
        "0e17aabbe6184e55921f84e979dcac43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3ff0bd22972346d0ad7f6ddb73d70d8b",
            "max": 50000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fb95c1d73f4e45c5b9146a407d3a0c76",
            "value": 50000
          }
        },
        "3525fdb2e81441c7b07cd064b6679b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d57e32bab614249a19307fddc4ea726",
            "placeholder": "​",
            "style": "IPY_MODEL_6b07e6a87e9a4007ba4abb9c03812604",
            "value": " 50000/50000 [32:55&lt;00:00, 25.31it/s]"
          }
        },
        "bbad5e3549ef4b6db1836e3531c07999": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "100%"
          }
        },
        "78407d86c2f9431cbda7098877acb6b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "267fc3ea8db74d4aaa797607c8010757": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ff0bd22972346d0ad7f6ddb73d70d8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb95c1d73f4e45c5b9146a407d3a0c76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d57e32bab614249a19307fddc4ea726": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b07e6a87e9a4007ba4abb9c03812604": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}