Lyric_Tutorial.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cZqKCBMVygPM",
        "colab_type": "text"
      },
      "source": [
        "# KorGPT2 Lyric Fine-Tuning Tutorial\n",
        "https://github.com/MrBananaHuman/KorGPT2Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pubr1XttzTLm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip -q install tqdm==4.46.0\n",
        "!pip -q install tokenizers==0.7.0\n",
        "!pip -q install torch==1.5.0\n",
        "!pip -q install transformers==2.11.0\n",
        "!pip -q install gdown"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "\u001b[33mYou are using pip version 19.0.3, however version 20.2b1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n\u001b[33mYou are using pip version 19.0.3, however version 20.2b1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n\u001b[33mYou are using pip version 19.0.3, however version 20.2b1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n\u001b[33mYou are using pip version 19.0.3, however version 20.2b1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n\u001b[33mYou are using pip version 19.0.3, however version 20.2b1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Download the pre-trained model\n",
        "* https://drive.google.com/drive/folders/124Uux07pym2YaCeQKQWNhzhLNeIlLm7r?usp=sharing "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98ytte2a0DRT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "655995d4-abb0-47f5-b888-24aa789d938f"
      },
      "source": [
        "!mkdir -p KorGPT-2SampleModel\n",
        "!gdown -O ./KorGPT-2SampleModel/pytorch_model.bin --id 1kX_dB05dkLRgxJkqoHidrT2OFYHGYWPF"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Downloading...\nFrom: https://drive.google.com/uc?id=1kX_dB05dkLRgxJkqoHidrT2OFYHGYWPF\nTo: /Users/hunkim/work/KorGPT2Tutorial/KorGPT-2SampleModel/pytorch_model.bin\n516MB [00:09, 55.3MB/s]\n"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "84WorptU8XxY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tokenizer code\n",
        "!pygmentize new_tokenizer.py"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mtokenizers\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mimplementations\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m SentencePieceBPETokenizer\n\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mtokenizers\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mprocessors\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m BertProcessing\n\n\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36mtransformers\u001b[39;49;00m\u001b[04m\u001b[36m.\u001b[39;49;00m\u001b[04m\u001b[36mtokenization_utils\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m PreTrainedTokenizer, PreTrainedTokenizerFast\n\n\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mjson\u001b[39;49;00m\n\n\u001b[34mclass\u001b[39;49;00m \u001b[04m\u001b[32mMyTokenizer\u001b[39;49;00m():\n\n    \u001b[34mdef\u001b[39;49;00m \u001b[32m__init__\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, vocab_file_path, merge_file_path):\n        \u001b[36mself\u001b[39;49;00m.tokenizer = SentencePieceBPETokenizer(vocab_file_path, merge_file_path)\n        \u001b[36mself\u001b[39;49;00m.unknown_token = \u001b[36mself\u001b[39;49;00m.tokenizer.token_to_id(\u001b[33m\"\u001b[39;49;00m\u001b[33m<unk>\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n        \u001b[36mself\u001b[39;49;00m._pad_token = \u001b[33m\"\u001b[39;49;00m\u001b[33m<pad>\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\n        \u001b[36mself\u001b[39;49;00m.pad_token_id = \u001b[36mself\u001b[39;49;00m.tokenizer.token_to_id(\u001b[33m\"\u001b[39;49;00m\u001b[33m<pad>\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n        \u001b[36mself\u001b[39;49;00m.max_len = \u001b[34m1024\u001b[39;49;00m\n        \u001b[36mself\u001b[39;49;00m.max_len_single_sentence = \u001b[34m1024\u001b[39;49;00m\n        \u001b[36mself\u001b[39;49;00m.init_kwargs = {}\n        \u001b[36mself\u001b[39;49;00m.added_tokens_encoder = {}\n        \u001b[36mself\u001b[39;49;00m.unique_added_tokens_encoder = \u001b[36mset\u001b[39;49;00m()\n        \u001b[36mself\u001b[39;49;00m.added_tokens_decoder = {}\n        \u001b[36mself\u001b[39;49;00m.unexpected_sep_token = [\u001b[33m'\u001b[39;49;00m\u001b[33m<pad>\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33m<unk>\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33m<eos>\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33m<sos>\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]\n\n        \u001b[36mself\u001b[39;49;00m.encoder = \u001b[36mself\u001b[39;49;00m.tokenizer.get_vocab()\n        \u001b[36mself\u001b[39;49;00m.decoder = \u001b[36mdict\u001b[39;49;00m(\u001b[36mmap\u001b[39;49;00m(\u001b[36mreversed\u001b[39;49;00m, \u001b[36mself\u001b[39;49;00m.encoder.items()))\n\n    \n    \u001b[34mdef\u001b[39;49;00m \u001b[32mtokenize\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, text):\n        \u001b[34mif\u001b[39;49;00m text \u001b[35min\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.unexpected_sep_token:\n            \u001b[34mreturn\u001b[39;49;00m text\n        \u001b[34mreturn\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.tokenizer.encode(text).tokens\n    \n    \u001b[34mdef\u001b[39;49;00m \u001b[32mconvert_tokens_to_ids\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, tokens):\n        ids = []\n        \u001b[34mif\u001b[39;49;00m \u001b[36misinstance\u001b[39;49;00m(tokens, \u001b[36mstr\u001b[39;49;00m):\n            \u001b[34mif\u001b[39;49;00m tokens \u001b[35min\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.encoder:\n                \u001b[34mreturn\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.encoder[tokens]\n            \u001b[34melse\u001b[39;49;00m:\n                \u001b[34mreturn\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.unknown_token\n        \u001b[34mfor\u001b[39;49;00m token \u001b[35min\u001b[39;49;00m tokens:\n            \u001b[34mif\u001b[39;49;00m token \u001b[35min\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.encoder:\n                ids.append(\u001b[36mself\u001b[39;49;00m.encoder[token])\n            \u001b[34melse\u001b[39;49;00m:\n                ids.append(\u001b[36mself\u001b[39;49;00m.unknown_token)\n        \u001b[34mreturn\u001b[39;49;00m ids\n    \n    \u001b[34mdef\u001b[39;49;00m \u001b[32mconvert_ids_to_tokens\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, ids):\n        sentence = \u001b[33m'\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\n        \u001b[34mfor\u001b[39;49;00m id_ \u001b[35min\u001b[39;49;00m ids:\n            sentence += \u001b[36mself\u001b[39;49;00m.decoder[id_]\n        sentence = sentence.replace(\u001b[33m'\u001b[39;49;00m\u001b[33m▁\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33m \u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\n        \u001b[34mreturn\u001b[39;49;00m sentence.strip()\n            \n    \n    \u001b[34mdef\u001b[39;49;00m \u001b[32mbuild_inputs_with_special_tokens\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, ids):\n        \u001b[34mreturn\u001b[39;49;00m ids\n\n    \u001b[34mdef\u001b[39;49;00m \u001b[32mget_vocab_size\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m):\n        \u001b[34mreturn\u001b[39;49;00m \u001b[36mself\u001b[39;49;00m.tokenizer.get_vocab_size()\n\n    \u001b[34mdef\u001b[39;49;00m \u001b[32madd_special_tokens\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, new_tokens):\n        \u001b[36mself\u001b[39;49;00m.tokenizer.add_special_tokens(new_tokens)\n        \u001b[36mself\u001b[39;49;00m.encoder = \u001b[36mself\u001b[39;49;00m.tokenizer.get_vocab()\n        \u001b[36mself\u001b[39;49;00m.decoder = \u001b[36mdict\u001b[39;49;00m(\u001b[36mmap\u001b[39;49;00m(\u001b[36mreversed\u001b[39;49;00m, \u001b[36mself\u001b[39;49;00m.encoder.items()))\n\n    \u001b[34mdef\u001b[39;49;00m \u001b[32madd_tokens\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, new_tokens):\n        \u001b[36mself\u001b[39;49;00m.tokenizer.add_tokens(new_tokens)\n        \u001b[36mself\u001b[39;49;00m.encoder = \u001b[36mself\u001b[39;49;00m.tokenizer.get_vocab()\n        \u001b[36mself\u001b[39;49;00m.decoder = \u001b[36mdict\u001b[39;49;00m(\u001b[36mmap\u001b[39;49;00m(\u001b[36mreversed\u001b[39;49;00m, \u001b[36mself\u001b[39;49;00m.encoder.items()))\n\n\u001b[34mif\u001b[39;49;00m \u001b[31m__name__\u001b[39;49;00m == \u001b[33m'\u001b[39;49;00m\u001b[33m__main__\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m:\n    vocab_file_path = \u001b[33m'\u001b[39;49;00m\u001b[33mtokenizer/vocab.json\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\n    merge_file_path = \u001b[33m'\u001b[39;49;00m\u001b[33mtokenizer/merges.txt\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\n    tokenizer = MyTokenizer(vocab_file_path, merge_file_path)\n    sentence = \u001b[33m\"\u001b[39;49;00m\u001b[33m이순신은 조선 중기의 무신이다.\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\n    tokens = tokenizer.tokenize(sentence)\n    \u001b[36mprint\u001b[39;49;00m(tokens)\n    ids = tokenizer.convert_tokens_to_ids(tokens)\n    \u001b[36mprint\u001b[39;49;00m(ids)\n    ids2 = tokenizer.build_inputs_with_special_tokens(ids)\n    \u001b[36mprint\u001b[39;49;00m(ids2)\n    \u001b[36mprint\u001b[39;49;00m(tokenizer.convert_ids_to_tokens(ids))\n    \n"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Loading the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJtKqUod96-l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9e72c78d-85c5-45b5-f70b-346ccb655344"
      },
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Config, AdamW\n",
        "from new_tokenizer import MyTokenizer\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "vocab_file_path = './tokenizer/vocab.json'\n",
        "merge_file_path = './tokenizer/merges.txt'\n",
        "model_dir = './KorGPT-2SampleModel/pytorch_model.bin'\n",
        "\n",
        "tokenizer = MyTokenizer(vocab_file_path, merge_file_path)\n",
        "config = GPT2Config(vocab_size=52000)\n",
        "model = GPT2LMHeadModel(config)\n",
        "  \n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model.load_state_dict(torch.load(model_dir, map_location=device), strict=False)\n",
        "model.to(device).eval()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": "GPT2LMHeadModel(\n  (transformer): GPT2Model(\n    (wte): Embedding(52000, 768)\n    (wpe): Embedding(1024, 768)\n    (drop): Dropout(p=0.1, inplace=False)\n    (h): ModuleList(\n      (0): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (1): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (2): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (3): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (4): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (5): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (6): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (7): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (8): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (9): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (10): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (11): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0.1, inplace=False)\n          (resid_dropout): Dropout(p=0.1, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n  )\n  (lm_head): Linear(in_features=768, out_features=52000, bias=False)\n)"
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "ATTR_TO_SPECIAL_TOKEN = ['<song>', '</song>']\n",
        "\n",
        "def add_special_tokens_(model, tokenizer):\n",
        "    orig_num_tokens = tokenizer.get_vocab_size()\n",
        "    tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN)\n",
        "    num_added_tokens = len(ATTR_TO_SPECIAL_TOKEN)\n",
        "    model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens + 1)\n",
        "\n",
        "add_special_tokens_(model, tokenizer)\n",
        "b_song = tokenizer.convert_tokens_to_ids('<song>')\n",
        "e_song = tokenizer.convert_tokens_to_ids('</song>')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Line by line dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jEDeF64x06sp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "5726620d-73af-4a40-adc8-f9737f3da192"
      },
      "source": [
        "class LyricDataSet(Dataset):\n",
        "    def __init__(self, file_path):\n",
        "        self.data = []\n",
        "        self.file_path = file_path\n",
        "        \n",
        "    def split_songs(self, lines):\n",
        "        songs = []\n",
        "        single_song = []\n",
        "        for line in lines:\n",
        "            line = line.strip()\n",
        "            if line == '':\n",
        "                if len(single_song) > 5:\n",
        "                    songs.append(single_song)\n",
        "                single_song = []\n",
        "            else:\n",
        "                single_song.append(line)\n",
        "        return songs\n",
        "    \n",
        "    def load_data(self):\n",
        "        lyric_file = open(self.file_path, 'r', encoding='utf-8')\n",
        "        lyric_lines = lyric_file.readlines()\n",
        "        lyric_file.close()\n",
        "        \n",
        "        song_list = self.split_songs(lyric_lines)\n",
        "        for song in song_list:\n",
        "            song_data = ['<song>']\n",
        "            for line in song:\n",
        "                tokenized_line = ['<s>'] + tokenizer.tokenize(line) + ['</s>']\n",
        "                if len(song_data) + len(tokenized_line) < 1024:\n",
        "                    song_data += tokenized_line\n",
        "                else:\n",
        "                    break\n",
        "            song_data += ['</song>']\n",
        "            padded_song_data = song_data + ['<pad>'] * (1024 - len(song_data))\n",
        "            self.data.append(torch.tensor(tokenizer.convert_tokens_to_ids(padded_song_data)).unsqueeze(0))\n",
        "                \n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        item = self.data[index]\n",
        "        return item\n",
        "\n",
        "\n",
        "        \n",
        "\n",
        "lyric_file_path = 'lyric_data/preprocessed_data.txt'\n",
        "lyric_data = LyricDataSet(lyric_file_path)\n",
        "lyric_data.load_data()\n",
        "lyric_data_loader = DataLoader(lyric_data, batch_size=4, shuffle=True)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Find tuning/training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "epoch no.0 train no.1  loss = 7.05945 avg_loss = 7.05945\nepoch no.1 train no.2  loss = 4.58444 avg_loss = 5.81573\nepoch no.2 train no.3  loss = 1.98967 avg_loss = 4.52753\nepoch no.3 train no.4  loss = 1.82610 avg_loss = 3.84196\nepoch no.4 train no.5  loss = 1.74568 avg_loss = 3.41424\n"
        }
      ],
      "source": [
        "optimizer = AdamW(model.parameters(), lr=1e-4, correct_bias=True)\n",
        "\n",
        "epochs = 5\n",
        "count = 0\n",
        "\n",
        "avg_loss = (0.0, 0.0)\n",
        "for epoch in range(epochs):\n",
        "\tfor data in lyric_data_loader:\n",
        "\t\toptimizer.zero_grad()\n",
        "\t\tdata = data.transpose(1,0)\n",
        "\t\tdata = data.to(device)\n",
        "\t\tmodel = model.to(device)\n",
        "\n",
        "\t\toutputs = model(data, labels=data)\n",
        "\t\tloss, logits = outputs[:2]\n",
        "\t\tloss = loss.to(device)\n",
        "\t\tloss.backward()\n",
        "\t\tavg_loss = (avg_loss[0] * 0.99 + loss, avg_loss[1] * 0.99 + 1.0)\n",
        "\t\toptimizer.step()\n",
        "\t\tcount+=1\n",
        "\n",
        "\tprint('epoch no.{0} train no.{1}  loss = {2:.5f} avg_loss = {3:.5f}' . format(epoch, count, loss, avg_loss[0] / avg_loss[1]))\n",
        "\t\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLDamFuxDHey",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "a6a28aad-0238-4ae1-928b-300552ff4a10"
      },
      "source": [
        "# Save the mode \n",
        "from os import path\n",
        "\n",
        "torch.save(model.state_dict(), \n",
        "    path.join(path.dirname(model_dir), 'my_lyric_model.bin'))"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GpyOorlVBw1n",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "ea57d317-4607-4ae9-eca7-d1ed82ff0a23"
      },
      "source": [
        "bos = tokenizer.convert_tokens_to_ids('<s>')\n",
        "eos = tokenizer.convert_tokens_to_ids('</s>')\n",
        "pad = tokenizer.convert_tokens_to_ids('<pad>')\n",
        "unk = tokenizer.convert_tokens_to_ids('<unk>')\n",
        "\n",
        "def add_special_tokens_(model, tokenizer):\n",
        "    orig_num_tokens = tokenizer.get_vocab_size()\n",
        "    tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN)\n",
        "    num_added_tokens = len(ATTR_TO_SPECIAL_TOKEN)\n",
        "    model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens + 1)\n",
        "\n",
        "add_special_tokens_(model, tokenizer)\n",
        "b_song = tokenizer.convert_tokens_to_ids('<song>')\n",
        "e_song = tokenizer.convert_tokens_to_ids('</song>')\n",
        "\n",
        "def encoding(text):\n",
        "    tokens = ['<song>', '<s>'] + tokenizer.tokenize(text)\n",
        "    return torch.tensor(tokenizer.convert_tokens_to_ids(tokens)).unsqueeze(0)\n",
        "\n",
        "def decoding(ids):\n",
        "    return tokenizer.convert_ids_to_tokens(ids[0])\n",
        "\n",
        "input_ids = encoding('하늘을 날아')\n",
        "\n",
        "sample_outputs = model.generate(\n",
        "    input_ids,\n",
        "    do_sample=True, \n",
        "    max_length=1024, \n",
        "    top_k=50, \n",
        "    top_p=0.95, \n",
        "    eos_token_id=e_song,\n",
        "    early_stopping=True,\n",
        "    bad_words_ids=[[unk]]\n",
        ")\n",
        "print(decoding(sample_outputs.tolist()))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "Setting `pad_token_id` to 52001 (first `eos_token_id`) to generate sequence\n<song><s> 하늘을 날아야 한다고 말한다.</s><s> 그리고 얼마 후 어느 날, 갑자기 </s><s> </s><s> 시몽, 마블의 첫 등장에 이어서 등장한 카드.</s><s> </s><s> 첫 등장은 4화부터 등장.</s><s> 시몽을 위해 시몽에게 도움을 요청할 때 시몽의 적인 효과를 사용해 시몽을 소환하는 것이 주된 포인트였다.</s><s> 시몽의 등장은  시몽의 등장이 아니라며 시몽의 등장을 위해 시몽에게 연락을 받고 시몽의 등장에 들어간다.</s><s> 시몽의 등장은 4화에서 시몽이 재등장한다.</s><s> 시몽은 재등장.</s><s> 시몽의 등장은 6화에서 시몽이 재등장한다.</s><s> 시몽이 재등장하면서 시몽을 재등장시킨다.</s><s> 시몽이 재등장하면서 시몽의 등장은 4화에서 시몽의 등장은 5화에서 시몽이 재등장한다.</s><s> 시몽의 등장은 6화에서 시몽에게 첫 등장이 확정되었다.</s><s> 시몽은 재등장했으나 시몽의 등장이 3화에서 시몽에게 첫 등장을 하게 되자 시몽의 등장이 확정되고 시몽이 재등장한다.</s><s> 시몽이 재등장하자 시몽은 재등장하지만 시몽에게 등장을 하지 않는다.</s><s> 시몽이 재등장한다.</s><s> 시몽은 재등장하지만 시몽이 재등장해 시몽이 재등장한다.</s><s> 시몽은 재등장한다.</s><s> 시몽에게 등장이지만 시몽의 등장은 5화에서 시몽을 재등장한다.</s><s> 시몽에게 등장은 12화에서 시몽에게 재등장을 하면서 시몽의 등장은 9화에서 시몽에게 등장은 12화, 시몽이 재등장한다.</s><s> 시몽이 재등장해 시몽이 재등장해 시몽을 재등장.</s><s> 시몽을 재등장하고 시몽을 재등장한다.</s><s> 시몽을 재등장해 시몽의 등장은 6화에서 시몽에게 처음으로 등장은 23화에서 시몽이 재등장해 시몽이 재등장해 시몽의 등장은 7화에서 시몽이 재등장해 시몽이 재등장해 시몽을 재등장.</s><s> 시몽이 재등장하며 시몽은 재등장하나 시몽이 재등장해 시몽을 재등장.</s><s> 시몽을 재등장.</s><s> 시몽이 재등장해 시몽을 재등장.</s><s> 시몽의 등장은 6화에서 시몽이 재등장해 시몽을 재등장.</s><s> 시몽은 재등장하지만 시몽을 재등장해 시몽을 재등장.</s><s> 시몽을 재등장해 시몽의 등장은 5화에서 시몽을 재등장.</s><s> 시몽은 재등장하지만 시몽이 재등장해 시몽이 재등장해 시몽을 재등장.</s><s> 시몽의 등장은 6화에서 시몽을 재등장.</s><s> 시몽에게 등장은 29화에서 시몽에게 재등장.</s><s> 시몽이 재등장해 시몽에게 등장은 7화에서 시몽의 등장이 확정되었다.</s><s> 시몽의 등장은 8화에서 시몽이 재등장하며 시몽의 등장은 8화에서 시몽이 재등장했다.</s><s> 시몽이 재등장해 시몽의 등장은 5화에서 시몽에게 재등장해 시몽의 등장은 6화에서 시몽에게 등장의 등장은 5화에서 시몽을 재등장.</s><s> 시몽의 등장은 9화에서 시몽이 재등장해 시몽이 재등장해 시몽이 재등장.</s><s> 시몽은 재등장해 시몽이 재등장해 시몽의 등장이 확정되었다.</s><s> 시몽이 재등장해 시몽을 재등장.</s><s> 시몽의 등장은 8화에서 시몽이 재등장해 시몽의 등장은 14화에서 시몽의 등장은 9화에서 시몽이 재등장.</s><s> 시몽에게 등장은 11화에서 시몽을 재등장.</s><s> 시몽의 등장은 9화에서 시몽이 재등장해 시몽의 등장은 11화에서 시몽이 재등장.</s><s> 시몽에게 등장은 10화에서 시몽이 재등장해 시몽의 등장은 11화에서 시몽에게 등장은 9화에서 시몽의 등장은 9화에서 시몽에게 재등장.</s><s> 시몽은 재등장.</s><s> 시몽을 재등장.</s><s> 시몽에게 등장은 11화에서 시몽이 재등장해 시몽의 등장이 확정되었다.</s><s> 시몽은 재등장이지만 시몽의 등장은 4화에서 시몽의 등장은 9화에서 시몽에게 재등장해 시몽의 등장은 7화에서 시몽의 등장은 9화에서 시몽의 등장은 9화의 등장은 11화에서 시몽에게 재등장해 시몽이 재등장해 시몽을 재등장.</s><s> 시몽과의 등장은 23화에서 시몽의 등장은 11화에서 시몽에게 등장을 해 시몽을 재등장해 시몽이 재등장해 시몽의\n"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Cf4QGV_-ByFY"
      },
      "source": [
        "## Decoding using the saved model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kce9wLqq-Yni",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "dff61ab7-5f56-4059-df89-b5cfa48f40b7"
      },
      "source": [
        "!mkdir -p KorGPT-2SampleModel\n",
        "!gdown -O ./KorGPT-2SampleModel/lyric_model.bin --id 1nopu647K2KwnMAc97CNL2owPKA4GsF22"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Downloading...\nFrom: https://drive.google.com/uc?id=1nopu647K2KwnMAc97CNL2owPKA4GsF22\nTo: /Users/hunkim/work/KorGPT2Tutorial/KorGPT-2SampleModel/lyric_model.bin\n516MB [00:16, 31.9MB/s]\n"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XLqQmOTIEZ03",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7a568465-49ec-4637-dd09-7acb3e079d26"
      },
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Config\n",
        "import torch\n",
        "\n",
        "ATTR_TO_SPECIAL_TOKEN = ['<song>', '</song>']\n",
        "\n",
        "vocab_file_path = './tokenizer/vocab.json'\n",
        "merge_file_path = './tokenizer/merges.txt'\n",
        "model_dir = './KorGPT-2SampleModel/lyric_model.bin'\n",
        "\n",
        "tokenizer = MyTokenizer(vocab_file_path, merge_file_path)\n",
        "bos = tokenizer.convert_tokens_to_ids('<s>')\n",
        "eos = tokenizer.convert_tokens_to_ids('</s>')\n",
        "pad = tokenizer.convert_tokens_to_ids('<pad>')\n",
        "unk = tokenizer.convert_tokens_to_ids('<unk>')\n",
        "\n",
        "config = GPT2Config(vocab_size=52003, resid_pdrop=0, embd_pdrop=0, attn_pdrop=0, summary_first_dropout=0)\n",
        "model = GPT2LMHeadModel(config)\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.load_state_dict(torch.load(model_dir, map_location=device), strict=False)\n",
        "model.to(device)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": "GPT2LMHeadModel(\n  (transformer): GPT2Model(\n    (wte): Embedding(52003, 768)\n    (wpe): Embedding(1024, 768)\n    (drop): Dropout(p=0, inplace=False)\n    (h): ModuleList(\n      (0): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (1): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (2): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (3): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (4): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (5): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (6): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (7): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (8): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (9): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (10): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n      (11): Block(\n        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (c_attn): Conv1D()\n          (c_proj): Conv1D()\n          (attn_dropout): Dropout(p=0, inplace=False)\n          (resid_dropout): Dropout(p=0, inplace=False)\n        )\n        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (c_fc): Conv1D()\n          (c_proj): Conv1D()\n          (dropout): Dropout(p=0, inplace=False)\n        )\n      )\n    )\n    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n  )\n  (lm_head): Linear(in_features=768, out_features=52003, bias=False)\n)"
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hyzhBjTt-ToD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "5a0a84ce-ab9c-4f9f-cdaf-969d5135694c"
      },
      "source": [
        "\n",
        "\n",
        "def add_special_tokens_(model, tokenizer):\n",
        "    orig_num_tokens = tokenizer.get_vocab_size()\n",
        "    tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN)\n",
        "    num_added_tokens = len(ATTR_TO_SPECIAL_TOKEN)\n",
        "    model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens + 1)\n",
        "\n",
        "add_special_tokens_(model, tokenizer)\n",
        "b_song = tokenizer.convert_tokens_to_ids('<song>')\n",
        "e_song = tokenizer.convert_tokens_to_ids('</song>')\n",
        "\n",
        "def encoding(text):\n",
        "    tokens = ['<song>', '<s>'] + tokenizer.tokenize(text)\n",
        "    return torch.tensor(tokenizer.convert_tokens_to_ids(tokens)).unsqueeze(0)\n",
        "\n",
        "def decoding(ids):\n",
        "    return tokenizer.convert_ids_to_tokens(ids[0])\n",
        "\n",
        "input_ids = encoding('우리는 오늘')\n",
        "\n",
        "sample_outputs = model.generate(\n",
        "    input_ids,\n",
        "    do_sample=True, \n",
        "    max_length=1024, \n",
        "    top_k=50, \n",
        "    top_p=0.95, \n",
        "    eos_token_id=e_song,\n",
        "    early_stopping=True,\n",
        "    bad_words_ids=[[unk]]\n",
        ")\n",
        "print(decoding(sample_outputs.tolist()))\n",
        "\n"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "Setting `pad_token_id` to 52001 (first `eos_token_id`) to generate sequence\n<song><s> 우리는 오늘 밤도 오늘 밤</s><s> 함께 걷던 이 곳</s><s> 우리 둘 사이엔 아직 많은 날들이</s><s> 우리 둘의 추억을 함께했던 추억은 사라져 버렸어</s><s> 우리의 사랑은 우리의 우정</s><s> 우리 둘이 함께했던 기억엔</s><s> 우리의 추억이 남아 있어요</s><s> 우리의 추억을 함께했던 시간엔</s><s> 우리의 추억이 남아 있어요</s><s> 우리 둘은 함께한 기억이 남아 있어요</s><s> 우리의 추억을 함께했던 시간엔</s><s> 우리의 추억이 남아 있어요</s><s> 우리의 추억을 함께했던 시간엔</s><s> 우리의 추억이 남아 있어요</s><s> 우리의 추억을 함께했던 시간엔</s><s> 우리의 추억이 남아 있어요</s></song>\n"
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "KorGPT2Tutorial",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python37364bitvenvvenvfefc9bbb02644ec2a36fa207a9f237c2",
      "display_name": "Python 3.7.3 64-bit ('venv': venv)"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}