tutorials/NL4Cell_Tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NL4Cell_Tutorial",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b6ad1c5873ad4ac187e72fe1083103cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bb65c99ad9de473db700cf0c76d7eeef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2cfffc72df6144f190f447fad8a8de32",
              "IPY_MODEL_d7991e8d332a41d688c7134dc69c050c",
              "IPY_MODEL_d0534187bce24c10aa6f9f9781dd67ee"
            ]
          }
        },
        "bb65c99ad9de473db700cf0c76d7eeef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2cfffc72df6144f190f447fad8a8de32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa87cad9ee9a47678e3fd757df6830ff",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ec2d1a63cd7546ffb4b18b87b88f9a37"
          }
        },
        "d7991e8d332a41d688c7134dc69c050c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e33f5ffa64e745c59d4adbace63e9f53",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8d5195f8200643c0890b0bda4307d9bb"
          }
        },
        "d0534187bce24c10aa6f9f9781dd67ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_871f36af5dfa4e2f9f4e35d6b8a057a3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/? [00:01&lt;00:00, 15.68 tables/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_360cfdc55716473fb9126a68f213f95a"
          }
        },
        "fa87cad9ee9a47678e3fd757df6830ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ec2d1a63cd7546ffb4b18b87b88f9a37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e33f5ffa64e745c59d4adbace63e9f53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8d5195f8200643c0890b0bda4307d9bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "871f36af5dfa4e2f9f4e35d6b8a057a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "360cfdc55716473fb9126a68f213f95a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d90c38cdc7943c1997a873e09b9fbd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4c0a469e5fbe471cbad4524fa19abe45",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a3231c64fd5543b4997021d576f7e8c7",
              "IPY_MODEL_38d82cd9bfbb45b881b3e19926ef694e",
              "IPY_MODEL_4fec9bee6fee4a09bc7be07dd665c50e"
            ]
          }
        },
        "4c0a469e5fbe471cbad4524fa19abe45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3231c64fd5543b4997021d576f7e8c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_33b49889c361428ca2a579ed1620c9d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21a7e37b32374b32a76b0410f0da2e65"
          }
        },
        "38d82cd9bfbb45b881b3e19926ef694e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2c240bc72a8d49eb96966defc671d0f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 31250,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 31250,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5e88a5b75044d94b4dd0ab110c09887"
          }
        },
        "4fec9bee6fee4a09bc7be07dd665c50e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eb721b7472be46b7857c75b1ff9fcf38",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 31250/31250 [02:41&lt;00:00, 186.34ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bed88db4c68749b09c6cfbc125757991"
          }
        },
        "33b49889c361428ca2a579ed1620c9d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21a7e37b32374b32a76b0410f0da2e65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c240bc72a8d49eb96966defc671d0f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5e88a5b75044d94b4dd0ab110c09887": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eb721b7472be46b7857c75b1ff9fcf38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bed88db4c68749b09c6cfbc125757991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/STRIDES-Codes/NL4Cell-Integrating-NLP-with-single-cell-data-analysis/blob/main/NL4Cell_Tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wJ4LiB_So76e"
      },
      "source": [
        "# Getting Started\n",
        "\n",
        "Welcome to NL4Cell! This interactive tutorial will take you all the way from data preprocessing through training our first model.\n",
        "\n",
        "To get started with this notebook, first we need to connect it to Google Drive; this way we have all of our data in one nice spot. Run the blocks of code below and follow the prompts.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGawA05dBnAA"
      },
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTM_xiFtOwZs",
        "outputId": "be646237-51ad-43a3-9403-498e8d6ecd8e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gkK3DOoa4lbj"
      },
      "source": [
        "Now that we've connected this notebook to Google drive, we can install the required packages. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mwNnvYmj0wJs"
      },
      "source": [
        "%%capture\n",
        "!pip install datasets"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WELNTlBBrQe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "30370351-4d09-4035-9f60-4357b7268a56"
      },
      "source": [
        "# %%capture\n",
        "!pip install --force-reinstall git+https://github.com/justinphan3110/transformers.git"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/justinphan3110/transformers.git\n",
            "  Cloning https://github.com/justinphan3110/transformers.git to /tmp/pip-req-build-g_d4etmu\n",
            "  Running command git clone -q https://github.com/justinphan3110/transformers.git /tmp/pip-req-build-g_d4etmu\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 9.0 MB/s \n",
            "\u001b[?25hCollecting numpy>=1.17\n",
            "  Downloading numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 15.7 MB 76 kB/s \n",
            "\u001b[?25hCollecting huggingface-hub==0.0.12\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Collecting pyyaml\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 39.6 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 47.5 MB/s \n",
            "\u001b[?25hCollecting regex!=2019.12.17\n",
            "  Downloading regex-2021.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (722 kB)\n",
            "\u001b[K     |████████████████████████████████| 722 kB 39.3 MB/s \n",
            "\u001b[?25hCollecting importlib-metadata\n",
            "  Downloading importlib_metadata-4.6.3-py3-none-any.whl (17 kB)\n",
            "Collecting filelock\n",
            "  Downloading filelock-3.0.12-py3-none-any.whl (7.6 kB)\n",
            "Collecting packaging\n",
            "  Downloading packaging-21.0-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 7.1 MB/s \n",
            "\u001b[?25hCollecting requests\n",
            "  Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n",
            "\u001b[K     |████████████████████████████████| 62 kB 1.2 MB/s \n",
            "\u001b[?25hCollecting tqdm>=4.27\n",
            "  Using cached tqdm-4.62.0-py2.py3-none-any.whl (76 kB)\n",
            "Collecting typing-extensions\n",
            "  Downloading typing_extensions-3.10.0.0-py3-none-any.whl (26 kB)\n",
            "Collecting pyparsing>=2.0.2\n",
            "  Downloading pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)\n",
            "\u001b[K     |████████████████████████████████| 67 kB 8.4 MB/s \n",
            "\u001b[?25hCollecting zipp>=0.5\n",
            "  Downloading zipp-3.5.0-py3-none-any.whl (5.7 kB)\n",
            "Collecting idna<4,>=2.5\n",
            "  Downloading idna-3.2-py3-none-any.whl (59 kB)\n",
            "\u001b[K     |████████████████████████████████| 59 kB 9.3 MB/s \n",
            "\u001b[?25hCollecting charset-normalizer~=2.0.0\n",
            "  Downloading charset_normalizer-2.0.4-py3-none-any.whl (36 kB)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.6-py2.py3-none-any.whl (138 kB)\n",
            "\u001b[K     |████████████████████████████████| 138 kB 54.9 MB/s \n",
            "\u001b[?25hCollecting certifi>=2017.4.17\n",
            "  Downloading certifi-2021.5.30-py2.py3-none-any.whl (145 kB)\n",
            "\u001b[K     |████████████████████████████████| 145 kB 75.9 MB/s \n",
            "\u001b[?25hCollecting joblib\n",
            "  Downloading joblib-1.0.1-py3-none-any.whl (303 kB)\n",
            "\u001b[K     |████████████████████████████████| 303 kB 65.5 MB/s \n",
            "\u001b[?25hCollecting six\n",
            "  Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting click\n",
            "  Downloading click-8.0.1-py3-none-any.whl (97 kB)\n",
            "\u001b[K     |████████████████████████████████| 97 kB 9.9 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.9.0.dev0-py3-none-any.whl size=2535279 sha256=3e081f55fd9791278ff821abcc7f9e5f93bf0e925746c47678121a124f1f7e9f\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-td7uq1_q/wheels/18/e4/e5/081c522c96f60fe34d3487499b8415b7c13a42d319b2e4a7af\n",
            "Successfully built transformers\n",
            "Installing collected packages: zipp, typing-extensions, urllib3, pyparsing, importlib-metadata, idna, charset-normalizer, certifi, tqdm, six, requests, regex, packaging, joblib, filelock, click, tokenizers, sacremoses, pyyaml, numpy, huggingface-hub, transformers\n",
            "  Attempting uninstall: zipp\n",
            "    Found existing installation: zipp 3.5.0\n",
            "    Uninstalling zipp-3.5.0:\n",
            "      Successfully uninstalled zipp-3.5.0\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing-extensions 3.7.4.3\n",
            "    Uninstalling typing-extensions-3.7.4.3:\n",
            "      Successfully uninstalled typing-extensions-3.7.4.3\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Attempting uninstall: pyparsing\n",
            "    Found existing installation: pyparsing 2.4.7\n",
            "    Uninstalling pyparsing-2.4.7:\n",
            "      Successfully uninstalled pyparsing-2.4.7\n",
            "  Attempting uninstall: importlib-metadata\n",
            "    Found existing installation: importlib-metadata 4.6.1\n",
            "    Uninstalling importlib-metadata-4.6.1:\n",
            "      Successfully uninstalled importlib-metadata-4.6.1\n",
            "  Attempting uninstall: idna\n",
            "    Found existing installation: idna 2.10\n",
            "    Uninstalling idna-2.10:\n",
            "      Successfully uninstalled idna-2.10\n",
            "  Attempting uninstall: charset-normalizer\n",
            "    Found existing installation: charset-normalizer 2.0.2\n",
            "    Uninstalling charset-normalizer-2.0.2:\n",
            "      Successfully uninstalled charset-normalizer-2.0.2\n",
            "  Attempting uninstall: certifi\n",
            "    Found existing installation: certifi 2021.5.30\n",
            "    Uninstalling certifi-2021.5.30:\n",
            "      Successfully uninstalled certifi-2021.5.30\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.62.0\n",
            "    Uninstalling tqdm-4.62.0:\n",
            "      Successfully uninstalled tqdm-4.62.0\n",
            "  Attempting uninstall: six\n",
            "    Found existing installation: six 1.15.0\n",
            "    Uninstalling six-1.15.0:\n",
            "      Successfully uninstalled six-1.15.0\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: regex\n",
            "    Found existing installation: regex 2019.12.20\n",
            "    Uninstalling regex-2019.12.20:\n",
            "      Successfully uninstalled regex-2019.12.20\n",
            "  Attempting uninstall: packaging\n",
            "    Found existing installation: packaging 21.0\n",
            "    Uninstalling packaging-21.0:\n",
            "      Successfully uninstalled packaging-21.0\n",
            "  Attempting uninstall: joblib\n",
            "    Found existing installation: joblib 1.0.1\n",
            "    Uninstalling joblib-1.0.1:\n",
            "      Successfully uninstalled joblib-1.0.1\n",
            "  Attempting uninstall: filelock\n",
            "    Found existing installation: filelock 3.0.12\n",
            "    Uninstalling filelock-3.0.12:\n",
            "      Successfully uninstalled filelock-3.0.12\n",
            "  Attempting uninstall: click\n",
            "    Found existing installation: click 7.1.2\n",
            "    Uninstalling click-7.1.2:\n",
            "      Successfully uninstalled click-7.1.2\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.19.5\n",
            "    Uninstalling numpy-1.19.5:\n",
            "      Successfully uninstalled numpy-1.19.5\n",
            "  Attempting uninstall: huggingface-hub\n",
            "    Found existing installation: huggingface-hub 0.0.15\n",
            "    Uninstalling huggingface-hub-0.0.15:\n",
            "      Successfully uninstalled huggingface-hub-0.0.15\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.5.0 requires numpy~=1.19.2, but you have numpy 1.21.1 which is incompatible.\n",
            "tensorflow 2.5.0 requires six~=1.15.0, but you have six 1.16.0 which is incompatible.\n",
            "tensorflow 2.5.0 requires typing-extensions~=3.7.4, but you have typing-extensions 3.10.0.0 which is incompatible.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n",
            "google-colab 1.0.0 requires six~=1.15.0, but you have six 1.16.0 which is incompatible.\n",
            "flask 1.1.4 requires click<8.0,>=5.1, but you have click 8.0.1 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed certifi-2021.5.30 charset-normalizer-2.0.4 click-8.0.1 filelock-3.0.12 huggingface-hub-0.0.12 idna-3.2 importlib-metadata-4.6.3 joblib-1.0.1 numpy-1.21.1 packaging-21.0 pyparsing-2.4.7 pyyaml-5.4.1 regex-2021.8.3 requests-2.26.0 sacremoses-0.0.45 six-1.16.0 tokenizers-0.10.3 tqdm-4.62.0 transformers-4.9.0.dev0 typing-extensions-3.10.0.0 urllib3-1.26.6 zipp-3.5.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "certifi",
                  "idna",
                  "numpy",
                  "pyparsing",
                  "requests",
                  "six",
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-xgtICZg4v0-"
      },
      "source": [
        "Now we handle the imports and make sure that we're using a GPU if it is available."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0W7MvbTjB2fp"
      },
      "source": [
        "from transformers import EncoderDecoderModel, BertTokenizer, BertConfig, BertModel\n",
        "import torch\n",
        "import numpy as np"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ypPLSYBqCTMU",
        "outputId": "74e961e7-4c79-435e-e00e-697e6d15bcb3"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "  device = torch.device('cuda')\n",
        "  print(torch.cuda.get_device_name())\n",
        "else:\n",
        "  device = torch.device('cpu')\n",
        "  "
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAa52ldgo-Hj"
      },
      "source": [
        "## Handling preprocessed data.\n",
        "\n",
        "To make our lives a little easier, we can start off by handling preprocessed data. \n",
        "\n",
        "The short version of what was done was that originally the data came in the form of a Pandas dataframe. Each row is an individual cell, and each column is a gene. Thus the dataframe created a gene expression matrix. Additional preprocessing was done, but for that see our more comprehensive [documentation](#null).\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XZU9Bv6i6QVZ"
      },
      "source": [
        "Let's start by coping the zipped dataframe into our space and expanding it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QEjAb9AI6Uqo"
      },
      "source": [
        "Taking a peek at this data, we can see that the matrix has numeric values for the gene expression data. This is more information than we need, so let's start by discritizing the data into positive and negative."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qN4eZVPqPYAl"
      },
      "source": [
        "## Preprocessed data\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IpDKwG7E9Dll"
      },
      "source": [
        "Each column (i.e. gene expression data for one gene across all cells in the data set) was selected and a threshold was determined using Otsu's technique. Cells with the specific gene expression value below this specific gene threshold were classified as negative (-), and genes above that threshold were classified as positive (+).\n",
        "\n",
        "These files were then converted to a text representation, pickled, and zipped, and we can access them now."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C3txBHgjPbqU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be77f4b9-77d9-45c2-955e-fe767e5eb703"
      },
      "source": [
        "!mkdir /content/drive/My\\ Drive/NL4Cell_data\n",
        "!gsutil -q -m cp -r gs://cytereader/*txt /content/drive/My\\ Drive/NL4Cell_data\n",
        "!gsutil -q -m cp -rv gs://cytereader/otsu /content/drive/My\\ Drive/NL4Cell_data\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mkdir: cannot create directory ‘/content/drive/My Drive/NL4Cell_data’: File exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-tWT3B2wTDD7"
      },
      "source": [
        "Now let's list out the directories to see if everything is there."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zjmGqkjeRv-n",
        "outputId": "293e8bf4-2401-4075-b43f-ad63d141b0c7"
      },
      "source": [
        "!ls /content/drive/My\\ Drive/NL4Cell_data\n",
        "!ls /content/drive/My\\ Drive/NL4Cell_data/otsu"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "otsu  vocab.txt\n",
            "vocab.txt\t\t Z2L2-otsu.3.txt\t Z3YR-otsu.12.txt\n",
            "Z23S-otsu.0.txt\t\t Z2L2-otsu.4.txt\t Z3YR-otsu.13.txt\n",
            "Z23S-otsu.10.txt\t Z2L2-otsu.5.txt\t Z3YR-otsu.14.txt\n",
            "Z23S-otsu.11.txt\t Z2L2-otsu.6.txt\t Z3YR-otsu.15.txt\n",
            "Z23S-otsu.12.txt\t Z2L2-otsu.7.txt\t Z3YR-otsu.16.txt\n",
            "Z23S-otsu.13.txt\t Z2L2-otsu.8.txt\t Z3YR-otsu.17.txt\n",
            "Z23S-otsu.14.txt\t Z367-otsu.0.txt\t Z3YR-otsu.18.txt\n",
            "Z23S-otsu.15.txt\t Z367-otsu.10.txt\t Z3YR-otsu.19.txt\n",
            "Z23S-otsu.16.txt_.gstmp  Z367-otsu.11.txt\t Z3YR-otsu.1.txt\n",
            "Z23S-otsu.1.txt\t\t Z367-otsu.1.txt\t Z3YR-otsu.20.txt\n",
            "Z23S-otsu.2.txt\t\t Z367-otsu.2.txt\t Z3YR-otsu.21.txt\n",
            "Z23S-otsu.3.txt\t\t Z367-otsu.3.txt\t Z3YR-otsu.22.txt\n",
            "Z23S-otsu.4.txt\t\t Z367-otsu.4.txt_.gstmp  Z3YR-otsu.23.txt\n",
            "Z23S-otsu.5.txt\t\t Z367-otsu.5.txt_.gstmp  Z3YR-otsu.2.txt\n",
            "Z23S-otsu.6.txt\t\t Z367-otsu.6.txt_.gstmp  Z3YR-otsu.3.txt\n",
            "Z23S-otsu.7.txt\t\t Z367-otsu.7.txt\t Z3YR-otsu.4.txt\n",
            "Z23S-otsu.8.txt\t\t Z367-otsu.8.txt\t Z3YR-otsu.5.txt\n",
            "Z23S-otsu.9.txt\t\t Z367-otsu.9.txt\t Z3YR-otsu.6.txt\n",
            "Z2L2-otsu.0.txt\t\t Z3YR-otsu.0.txt\t Z3YR-otsu.7.txt\n",
            "Z2L2-otsu.1.txt\t\t Z3YR-otsu.10.txt\t Z3YR-otsu.8.txt\n",
            "Z2L2-otsu.2.txt\t\t Z3YR-otsu.11.txt\t Z3YR-otsu.9.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9cLe_wk5Jcne"
      },
      "source": [
        "Now let's take a peek at one of the files..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dlnCTd0iJbC6",
        "outputId": "21dcb2db-ea1a-420a-d7fe-096d6c05b745"
      },
      "source": [
        "!head /content/drive/My\\ Drive/NL4Cell_data/otsu/Z23S-otsu.0.txt"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "igd- cd19- cd45ra- cd141+ cd4+ cd8- cd16- cd127+ cd1c- cd123- cd66b- cd27+ cd14- cd56- cd24- cd3+ cd38+ cd161+ cd25-\n",
            "igd- cd19- cd45ra+ cd141+ cd4- cd8+ cd16- cd127+ cd1c- cd123+ cd66b- cd27+ cd14- cd56- cd24+ cd3+ cd38- cd161+ cd25-\n",
            "igd- cd19- cd45ra+ cd141- cd4+ cd8- cd16- cd127+ cd1c- cd123- cd66b- cd27- cd14- cd56+ cd24- cd3+ cd38- cd161- cd25-\n",
            "igd- cd19- cd45ra+ cd141- cd4- cd8- cd16- cd127+ cd1c- cd123- cd66b- cd27+ cd14- cd56+ cd24+ cd3+ cd38- cd161- cd25-\n",
            "igd- cd19- cd45ra- cd141- cd4+ cd8- cd16- cd127+ cd1c- cd123- cd66b- cd27+ cd14- cd56- cd24- cd3+ cd38+ cd161- cd25-\n",
            "igd- cd19- cd45ra+ cd141- cd4+ cd8- cd16- cd127- cd1c- cd123- cd66b- cd27- cd14- cd56+ cd24- cd3+ cd38+ cd161- cd25-\n",
            "igd- cd19- cd45ra- cd141+ cd4+ cd8- cd16+ cd127- cd1c+ cd123- cd66b- cd27- cd14+ cd56- cd24- cd3- cd38+ cd161- cd25-\n",
            "igd- cd19- cd45ra- cd141+ cd4+ cd8- cd16- cd127+ cd1c- cd123- cd66b- cd27+ cd14- cd56- cd24+ cd3+ cd38- cd161- cd25-\n",
            "igd- cd19- cd45ra+ cd141+ cd4+ cd8- cd16- cd127- cd1c- cd123- cd66b- cd27- cd14- cd56+ cd24- cd3+ cd38- cd161- cd25-\n",
            "igd- cd19- cd45ra+ cd141+ cd4+ cd8- cd16- cd127- cd1c- cd123- cd66b- cd27- cd14- cd56- cd24- cd3+ cd38+ cd161- cd25-\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gnQ7ylPBVXMx"
      },
      "source": [
        "Each row in this example is a \"sentence\" as we would normally see it in NLP, and each gene is a word. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ElCbAC3sVpdK"
      },
      "source": [
        "Additionally we can take a look at `vocab.txt`. This file contains every gene (we'll use a placeholder, `gene`) across all of the data with three variants:\n",
        "* `gene`\n",
        "* `gene-`\n",
        "* `gene+`\n",
        "\n",
        "The `gene+` and `gene-` are values as we'd see them in the sentence, whereas the neutral `gene` is used for training-- it is a placeholder so that we can predict whether a specific gene is positive or negative without the model throwing in any random gene.\n",
        "\n",
        "Additionally it contains a number of spetial tokens, but this is more important down the road."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "va8rWupfWbzW",
        "outputId": "5c40b28e-4d80-4590-a8eb-7497600c5f1f"
      },
      "source": [
        "!head -n15 /content/drive/My\\ Drive/NL4Cell_data/vocab.txt"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[PAD]\n",
            "[UNK]\n",
            "[CLS]\n",
            "[SEP]\n",
            "[MASK]\n",
            "ccr4\n",
            "ccr4-\n",
            "ccr4+\n",
            "ccr5\n",
            "ccr5-\n",
            "ccr5+\n",
            "ccr6\n",
            "ccr6-\n",
            "ccr6+\n",
            "ccr7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tCcNyIxdW0Cs"
      },
      "source": [
        "And that's it for what the data looks like. There are a couple different ways to prepare the data (binary vs. stratified encoding, different thresholding methods), so for more information check out the relevant notebook."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7wurB7oMpAPh"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a1Y0VnKNXARy"
      },
      "source": [
        "Now that we've taken a look at the data we'll be using, we can start training. First we need to download a package that helps us tokenize the data.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3iJekoKc7P7h",
        "outputId": "82b6ebd3-eb98-49df-cba6-7ece8dc59aad"
      },
      "source": [
        "!pip install tokenizers"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tokenizers in /usr/local/lib/python3.7/dist-packages (0.10.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xbE3kS9FXaUi"
      },
      "source": [
        "Then we configure a modified version of Bert to be able to processes these gene \"sentences\" via regular tokenization methods. The model is modified so that the positional encoding doesn't matter (since `cd4+ cd8-` is the exact same as `cd8- cd4+`)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gOy5ACECNN0y",
        "outputId": "b4b4114e-c0a5-4be6-e8ef-3a39cc46b151"
      },
      "source": [
        "from transformers import BertConfig, BertTokenizer, BertLMHeadModel\n",
        "from transformers.models.bert.tokenization_bert import CellBertTokenizer\n",
        "\n",
        "configuration = BertConfig(vocab_size=1000)\n",
        "model = BertLMHeadModel(configuration)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "If you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rvx_GJWMYJ4I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e789a51a-100f-4d15-9b8f-afe4c938e84d"
      },
      "source": [
        "!mkdir cellAttention\n",
        "\n",
        "!gsutil cp gs://cytereader/vocab.txt cellAttention/"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Copying gs://cytereader/vocab.txt...\n",
            "/ [0 files][    0.0 B/  978.0 B]                                                \r/ [1 files][  978.0 B/  978.0 B]                                                \r\n",
            "Operation completed over 1 objects/978.0 B.                                      \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p8hJP_WSX8Gg"
      },
      "source": [
        "The `tokenizer` just converts the whole vocabulary into something that is numerically encoded. See the next cell for an example"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EsROp3Se1QeA"
      },
      "source": [
        "tokenizer = CellBertTokenizer.from_pretrained('./cellAttention',vocab_file=\"./cellAttention/vocab.txt\")\n"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A0FucsdeD7it",
        "outputId": "ad2e75f9-3b59-44c9-c00b-1dc0da37c38e"
      },
      "source": [
        "tokenizer.encode('CD45+ CD196_CCR6+'.lower())"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 97, 1, 3]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qjASCgUlYfVk"
      },
      "source": [
        "Now it's time to begin training on a small subset of encoded data. First we need to set up the tokenizer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 154,
          "referenced_widgets": [
            "b6ad1c5873ad4ac187e72fe1083103cb",
            "bb65c99ad9de473db700cf0c76d7eeef",
            "2cfffc72df6144f190f447fad8a8de32",
            "d7991e8d332a41d688c7134dc69c050c",
            "d0534187bce24c10aa6f9f9781dd67ee",
            "fa87cad9ee9a47678e3fd757df6830ff",
            "ec2d1a63cd7546ffb4b18b87b88f9a37",
            "e33f5ffa64e745c59d4adbace63e9f53",
            "8d5195f8200643c0890b0bda4307d9bb",
            "871f36af5dfa4e2f9f4e35d6b8a057a3",
            "360cfdc55716473fb9126a68f213f95a",
            "0d90c38cdc7943c1997a873e09b9fbd3",
            "4c0a469e5fbe471cbad4524fa19abe45",
            "a3231c64fd5543b4997021d576f7e8c7",
            "38d82cd9bfbb45b881b3e19926ef694e",
            "4fec9bee6fee4a09bc7be07dd665c50e",
            "33b49889c361428ca2a579ed1620c9d5",
            "21a7e37b32374b32a76b0410f0da2e65",
            "2c240bc72a8d49eb96966defc671d0f0",
            "d5e88a5b75044d94b4dd0ab110c09887",
            "eb721b7472be46b7857c75b1ff9fcf38",
            "bed88db4c68749b09c6cfbc125757991"
          ]
        },
        "id": "27tNvSbD1i0Z",
        "outputId": "75403d6e-9479-4d07-e0e6-a9c0bd98db1d"
      },
      "source": [
        "%%time\n",
        "from transformers import LineByLineTextDataset\n",
        "from datasets import load_dataset\n",
        "import os \n",
        "\n",
        "files_name = ['/content/drive/My Drive/NL4Cell_data/otsu/' + x for x in os.listdir('/content/drive/My Drive/NL4Cell_data/otsu')][0:2]\n",
        "dataset = load_dataset(\"text\", data_files=files_name, split='train')\n",
        "max_length = 128\n",
        "batch_size=64 \n",
        "\n",
        "def tokenize_function(examples):\n",
        "    tokenizer_ = tokenizer([x.lower() for x in examples['text']], max_length=max_length, truncation=True, padding='max_length')\n",
        "    return tokenizer_\n",
        "\n",
        "\n",
        "\n",
        "train_data_batch = dataset.map(\n",
        "    tokenize_function, \n",
        "    batched=True, \n",
        "    batch_size=batch_size,\n",
        ")\n"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using custom data configuration default-d2ed2ec12f0b89a9\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/text/default-d2ed2ec12f0b89a9/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b6ad1c5873ad4ac187e72fe1083103cb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 tables [00:00, ? tables/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Dataset text downloaded and prepared to /root/.cache/huggingface/datasets/text/default-d2ed2ec12f0b89a9/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0d90c38cdc7943c1997a873e09b9fbd3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/31250 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 2min 39s, sys: 5.2 s, total: 2min 44s\n",
            "Wall time: 2min 48s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "38_O4RByZA84"
      },
      "source": [
        "Then set up the data collator which helps form batches."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sDvACIyx2l73"
      },
      "source": [
        "from transformers.data.data_collator import DataCollatorForLanguageModeling, DataCollatorForSOP, DataCollatorForNetutralCellModeling\n",
        "\n",
        "data_collator = DataCollatorForNetutralCellModeling(\n",
        "    tokenizer=tokenizer, ncm=True, mlm_probability=0.15\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XD2nGyXFZGwu"
      },
      "source": [
        "\n",
        "Now we define the arguments for training..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fn_oz7tu9Q13"
      },
      "source": [
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./cellAttention\",\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=1,\n",
        "    per_device_train_batch_size=32,\n",
        "    save_steps=10_000,\n",
        "    learning_rate=1e-4,\n",
        "    save_total_limit=2,\n",
        "    prediction_loss_only=True,\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_data_batch,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vLp1dk_OZKnj"
      },
      "source": [
        "And train!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_mKjdOV9UF8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d3d4f38-ea14-4f02-a3c3-00c4f9907136"
      },
      "source": [
        "# %%time\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The following columns in the training set  don't have a corresponding argument in `BertLMHeadModel.forward` and have been ignored: text.\n",
            "***** Running training *****\n",
            "  Num examples = 2000000\n",
            "  Num Epochs = 1\n",
            "  Instantaneous batch size per device = 32\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 62500\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='23458' max='62500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [23458/62500 2:52:52 < 4:47:45, 2.26 it/s, Epoch 0.38/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.737100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.493400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.446900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.435100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.433400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.422900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.424100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.414000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.407900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.408700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.403200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.404600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.406900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.400900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.389900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.396400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.403600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.401900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.399600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.389000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.397400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.394300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.395600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.379400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.403400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.383700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.410200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.408900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.394000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.376200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.374700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.385400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.401000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.400400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.381300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.396800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.393600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.385900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.377500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20500</td>\n",
              "      <td>0.398600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>21000</td>\n",
              "      <td>0.377000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>21500</td>\n",
              "      <td>0.375600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>22000</td>\n",
              "      <td>0.400300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>22500</td>\n",
              "      <td>0.384700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>23000</td>\n",
              "      <td>0.374800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./cellAttention/checkpoint-10000\n",
            "Configuration saved in ./cellAttention/checkpoint-10000/config.json\n",
            "Model weights saved in ./cellAttention/checkpoint-10000/pytorch_model.bin\n",
            "Saving model checkpoint to ./cellAttention/checkpoint-20000\n",
            "Configuration saved in ./cellAttention/checkpoint-20000/config.json\n",
            "Model weights saved in ./cellAttention/checkpoint-20000/pytorch_model.bin\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GiwqahoUZNvE"
      },
      "source": [
        "Below this is just lines of code which you can play around with. First we save the model, then establish a pipeline which lets us predict which values are in a mask. For example, if we were dealing with a typical NLP model we could make it fill in the mask like so:\n",
        "\n",
        "`the chef [MASK] the meal` --> `the chef prepared the meal`. \n",
        "\n",
        "Likewise, we can pass a sequence of genes and have it try to infer genes that would fit well in its place.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNfsthQO9WYk"
      },
      "source": [
        "trainer.save_model(\"./cellAttention\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KaQacVClEQP8"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "fill_mask = pipeline(\n",
        "    \"fill-mask\",\n",
        "    model=\"./cellAttention\",\n",
        "    tokenizer=\"./cellAttention\"\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DD9HgAahZ8yZ"
      },
      "source": [
        "Predict if a cell corresponding to this vector will express `CD182_CXCR2`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b-Cg3tatEZuT"
      },
      "source": [
        "fill_mask(\"CD45+ CD196_CCR6+ CD181_CXCR1- HLA_DR- CD15- CD31_PECAM1- CD8a- CD182_CXCR2[MASK] CD66ace- CD63- CD14- CD66b- CD62L_Lselectin- CD3+ CD27- CD86+ CD10- CD197_CCR7+ CD28- CD11c- CD33- CD161- CD45RO- CD24- CD38+ CD278_ICOS- CD32- CD152_CTLA4+ IgM+ CD184_CXCR4+ CD279_PD1- CD56+ CD16-\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VEZM0hnklOg3"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xJvPGnhuaJ7x"
      },
      "source": [
        "And that's it for the tutorial! Now you have a model that understands genes like a normal NLP model would understand words."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xe0MkVF4aRLM"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}