material/deep_learning/rnn_text_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "rnn_text_classification.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gTgtpfC4eHuP"
      },
      "source": [
        "# 텍스트 분류\n",
        "\n",
        "copied and modified from https://www.tensorflow.org/tutorials/keras/text_classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-WERUuDeQCLq"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVZ8aq96QC0Y"
      },
      "source": [
        "## 설정\n",
        "VOCA_SIZE = 4000 # 어휘 사전의 크기\n",
        "EMBEDDING_SIZE = 64 # 단어를 임베딩한 벡터 크기"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j4ehmkWmN80W"
      },
      "source": [
        "# 데이터"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9xeAkpUYJPpy"
      },
      "source": [
        "## 데이터 로딩"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-nglK_9Hd3_r",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7fb61882-6cf0-4173-c428-fe0045201ee8"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "print('Loading data...')\n",
        "(train_x, train_y), (test_x, test_y) = tf.keras.datasets.imdb.load_data(num_words=VOCA_SIZE)\n",
        "\n",
        "print(train_x.shape)\n",
        "print(train_y.shape)\n",
        "print(test_x.shape)\n",
        "print(test_y.shape)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading data...\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
            "17465344/17464789 [==============================] - 0s 0us/step\n",
            "17473536/17464789 [==============================] - 0s 0us/step\n",
            "(25000,)\n",
            "(25000,)\n",
            "(25000,)\n",
            "(25000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2B3TWo3_JR2U"
      },
      "source": [
        "## 데이터 보기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wTUxzKwFJigM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "802f4c1a-2e70-4ad7-8733-e694deb0dae4"
      },
      "source": [
        "print(train_x[:5])\n",
        "print(train_y[:5])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[list([1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65, 458, 2, 66, 3941, 4, 173, 36, 256, 5, 25, 100, 43, 838, 112, 50, 670, 2, 9, 35, 480, 284, 5, 150, 4, 172, 112, 167, 2, 336, 385, 39, 4, 172, 2, 1111, 17, 546, 38, 13, 447, 4, 192, 50, 16, 6, 147, 2025, 19, 14, 22, 4, 1920, 2, 469, 4, 22, 71, 87, 12, 16, 43, 530, 38, 76, 15, 13, 1247, 4, 22, 17, 515, 17, 12, 16, 626, 18, 2, 5, 62, 386, 12, 8, 316, 8, 106, 5, 4, 2223, 2, 16, 480, 66, 3785, 33, 4, 130, 12, 16, 38, 619, 5, 25, 124, 51, 36, 135, 48, 25, 1415, 33, 6, 22, 12, 215, 28, 77, 52, 5, 14, 407, 16, 82, 2, 8, 4, 107, 117, 2, 15, 256, 4, 2, 7, 3766, 5, 723, 36, 71, 43, 530, 476, 26, 400, 317, 46, 7, 4, 2, 1029, 13, 104, 88, 4, 381, 15, 297, 98, 32, 2071, 56, 26, 141, 6, 194, 2, 18, 4, 226, 22, 21, 134, 476, 26, 480, 5, 144, 30, 2, 18, 51, 36, 28, 224, 92, 25, 104, 4, 226, 65, 16, 38, 1334, 88, 12, 16, 283, 5, 16, 2, 113, 103, 32, 15, 16, 2, 19, 178, 32])\n",
            " list([1, 194, 1153, 194, 2, 78, 228, 5, 6, 1463, 2, 2, 134, 26, 4, 715, 8, 118, 1634, 14, 394, 20, 13, 119, 954, 189, 102, 5, 207, 110, 3103, 21, 14, 69, 188, 8, 30, 23, 7, 4, 249, 126, 93, 4, 114, 9, 2300, 1523, 5, 647, 4, 116, 9, 35, 2, 4, 229, 9, 340, 1322, 4, 118, 9, 4, 130, 2, 19, 4, 1002, 5, 89, 29, 952, 46, 37, 4, 455, 9, 45, 43, 38, 1543, 1905, 398, 4, 1649, 26, 2, 5, 163, 11, 3215, 2, 4, 1153, 9, 194, 775, 7, 2, 2, 349, 2637, 148, 605, 2, 2, 15, 123, 125, 68, 2, 2, 15, 349, 165, 2, 98, 5, 4, 228, 9, 43, 2, 1157, 15, 299, 120, 5, 120, 174, 11, 220, 175, 136, 50, 9, 2, 228, 2, 5, 2, 656, 245, 2350, 5, 4, 2, 131, 152, 491, 18, 2, 32, 2, 1212, 14, 9, 6, 371, 78, 22, 625, 64, 1382, 9, 8, 168, 145, 23, 4, 1690, 15, 16, 4, 1355, 5, 28, 6, 52, 154, 462, 33, 89, 78, 285, 16, 145, 95])\n",
            " list([1, 14, 47, 8, 30, 31, 7, 4, 249, 108, 7, 4, 2, 54, 61, 369, 13, 71, 149, 14, 22, 112, 4, 2401, 311, 12, 16, 3711, 33, 75, 43, 1829, 296, 4, 86, 320, 35, 534, 19, 263, 2, 1301, 4, 1873, 33, 89, 78, 12, 66, 16, 4, 360, 7, 4, 58, 316, 334, 11, 4, 1716, 43, 645, 662, 8, 257, 85, 1200, 42, 1228, 2578, 83, 68, 3912, 15, 36, 165, 1539, 278, 36, 69, 2, 780, 8, 106, 14, 2, 1338, 18, 6, 22, 12, 215, 28, 610, 40, 6, 87, 326, 23, 2300, 21, 23, 22, 12, 272, 40, 57, 31, 11, 4, 22, 47, 6, 2307, 51, 9, 170, 23, 595, 116, 595, 1352, 13, 191, 79, 638, 89, 2, 14, 9, 8, 106, 607, 624, 35, 534, 6, 227, 7, 129, 113])\n",
            " list([1, 4, 2, 2, 33, 2804, 4, 2040, 432, 111, 153, 103, 4, 1494, 13, 70, 131, 67, 11, 61, 2, 744, 35, 3715, 761, 61, 2, 452, 2, 4, 985, 7, 2, 59, 166, 4, 105, 216, 1239, 41, 1797, 9, 15, 7, 35, 744, 2413, 31, 8, 4, 687, 23, 4, 2, 2, 6, 3693, 42, 38, 39, 121, 59, 456, 10, 10, 7, 265, 12, 575, 111, 153, 159, 59, 16, 1447, 21, 25, 586, 482, 39, 4, 96, 59, 716, 12, 4, 172, 65, 9, 579, 11, 2, 4, 1615, 5, 2, 7, 2, 17, 13, 2, 12, 19, 6, 464, 31, 314, 11, 2, 6, 719, 605, 11, 8, 202, 27, 310, 4, 3772, 3501, 8, 2722, 58, 10, 10, 537, 2116, 180, 40, 14, 413, 173, 7, 263, 112, 37, 152, 377, 4, 537, 263, 846, 579, 178, 54, 75, 71, 476, 36, 413, 263, 2504, 182, 5, 17, 75, 2306, 922, 36, 279, 131, 2895, 17, 2867, 42, 17, 35, 921, 2, 192, 5, 1219, 3890, 19, 2, 217, 2, 1710, 537, 2, 1236, 5, 736, 10, 10, 61, 403, 9, 2, 40, 61, 2, 5, 27, 2, 159, 90, 263, 2311, 2, 309, 8, 178, 5, 82, 2, 4, 65, 15, 2, 145, 143, 2, 12, 2, 537, 746, 537, 537, 15, 2, 4, 2, 594, 7, 2, 94, 2, 3987, 2, 11, 2, 4, 538, 7, 1795, 246, 2, 9, 2, 11, 635, 14, 9, 51, 408, 12, 94, 318, 1382, 12, 47, 6, 2683, 936, 5, 2, 2, 19, 49, 7, 4, 1885, 2, 1118, 25, 80, 126, 842, 10, 10, 2, 2, 2, 27, 2, 11, 1550, 3633, 159, 27, 341, 29, 2733, 19, 2, 173, 7, 90, 2, 8, 30, 11, 4, 1784, 86, 1117, 8, 3261, 46, 11, 2, 21, 29, 9, 2841, 23, 4, 1010, 2, 793, 6, 2, 1386, 1830, 10, 10, 246, 50, 9, 6, 2750, 1944, 746, 90, 29, 2, 8, 124, 4, 882, 4, 882, 496, 27, 2, 2213, 537, 121, 127, 1219, 130, 5, 29, 494, 8, 124, 4, 882, 496, 4, 341, 7, 27, 846, 10, 10, 29, 9, 1906, 8, 97, 6, 236, 2, 1311, 8, 4, 2, 7, 31, 7, 2, 91, 2, 3987, 70, 4, 882, 30, 579, 42, 9, 12, 32, 11, 537, 10, 10, 11, 14, 65, 44, 537, 75, 2, 1775, 3353, 2, 1846, 4, 2, 7, 154, 5, 4, 518, 53, 2, 2, 7, 3211, 882, 11, 399, 38, 75, 257, 3807, 19, 2, 17, 29, 456, 4, 65, 7, 27, 205, 113, 10, 10, 2, 4, 2, 2, 9, 242, 4, 91, 1202, 2, 5, 2070, 307, 22, 7, 2, 126, 93, 40, 2, 13, 188, 1076, 3222, 19, 4, 2, 7, 2348, 537, 23, 53, 537, 21, 82, 40, 2, 13, 2, 14, 280, 13, 219, 4, 2, 431, 758, 859, 4, 953, 1052, 2, 7, 2, 5, 94, 40, 25, 238, 60, 2, 4, 2, 804, 2, 7, 4, 2, 132, 8, 67, 6, 22, 15, 9, 283, 8, 2, 14, 31, 9, 242, 955, 48, 25, 279, 2, 23, 12, 1685, 195, 25, 238, 60, 796, 2, 4, 671, 7, 2804, 5, 4, 559, 154, 888, 7, 726, 50, 26, 49, 2, 15, 566, 30, 579, 21, 64, 2574])\n",
            " list([1, 249, 1323, 7, 61, 113, 10, 10, 13, 1637, 14, 20, 56, 33, 2401, 18, 457, 88, 13, 2626, 1400, 45, 3171, 13, 70, 79, 49, 706, 919, 13, 16, 355, 340, 355, 1696, 96, 143, 4, 22, 32, 289, 7, 61, 369, 71, 2359, 5, 13, 16, 131, 2073, 249, 114, 249, 229, 249, 20, 13, 28, 126, 110, 13, 473, 8, 569, 61, 419, 56, 429, 6, 1513, 18, 35, 534, 95, 474, 570, 5, 25, 124, 138, 88, 12, 421, 1543, 52, 725, 2, 61, 419, 11, 13, 1571, 15, 1543, 20, 11, 4, 2, 5, 296, 12, 3524, 5, 15, 421, 128, 74, 233, 334, 207, 126, 224, 12, 562, 298, 2167, 1272, 7, 2601, 5, 516, 988, 43, 8, 79, 120, 15, 595, 13, 784, 25, 3171, 18, 165, 170, 143, 19, 14, 5, 2, 6, 226, 251, 7, 61, 113])]\n",
            "[1 0 0 1 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ifl7K71wNchw"
      },
      "source": [
        "## 텍스트로 데이터 보기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mkw9MdgJLZOX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4450eace-8678-40bc-d0ac-a33a7d3a9fee"
      },
      "source": [
        "# 단어와 정수 인덱스를 매핑한 딕셔너리\n",
        "word_index = tf.keras.datasets.imdb.get_word_index()\n",
        "# word_index = {'fawn': 34701, 'tsukino': 52006, 'nunnery': 52007, ... }\n",
        "\n",
        "# 처음 몇 개 인덱스는 사전에 정의되어 있습니다\n",
        "word_index = {k:(v+3) for k,v in word_index.items()}\n",
        "word_index[\"<PAD>\"] = 0\n",
        "word_index[\"<START>\"] = 1\n",
        "word_index[\"<UNK>\"] = 2  # unknown\n",
        "word_index[\"<UNUSED>\"] = 3\n",
        "\n",
        "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
        "# reverse_word_index = {34704: 'fawn', 52009: 'tsukino', 52010: 'nunnery', ... }\n",
        "\n",
        "def decode_review(text):\n",
        "    return ' '.join([reverse_word_index.get(i, '?') for i in text])\n",
        "\n",
        "print(train_x[0])\n",
        "print(decode_review(train_x[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json\n",
            "1646592/1641221 [==============================] - 0s 0us/step\n",
            "1654784/1641221 [==============================] - 0s 0us/step\n",
            "[1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65, 458, 2, 66, 3941, 4, 173, 36, 256, 5, 25, 100, 43, 838, 112, 50, 670, 2, 9, 35, 480, 284, 5, 150, 4, 172, 112, 167, 2, 336, 385, 39, 4, 172, 2, 1111, 17, 546, 38, 13, 447, 4, 192, 50, 16, 6, 147, 2025, 19, 14, 22, 4, 1920, 2, 469, 4, 22, 71, 87, 12, 16, 43, 530, 38, 76, 15, 13, 1247, 4, 22, 17, 515, 17, 12, 16, 626, 18, 2, 5, 62, 386, 12, 8, 316, 8, 106, 5, 4, 2223, 2, 16, 480, 66, 3785, 33, 4, 130, 12, 16, 38, 619, 5, 25, 124, 51, 36, 135, 48, 25, 1415, 33, 6, 22, 12, 215, 28, 77, 52, 5, 14, 407, 16, 82, 2, 8, 4, 107, 117, 2, 15, 256, 4, 2, 7, 3766, 5, 723, 36, 71, 43, 530, 476, 26, 400, 317, 46, 7, 4, 2, 1029, 13, 104, 88, 4, 381, 15, 297, 98, 32, 2071, 56, 26, 141, 6, 194, 2, 18, 4, 226, 22, 21, 134, 476, 26, 480, 5, 144, 30, 2, 18, 51, 36, 28, 224, 92, 25, 104, 4, 226, 65, 16, 38, 1334, 88, 12, 16, 283, 5, 16, 2, 113, 103, 32, 15, 16, 2, 19, 178, 32]\n",
            "<START> this film was just brilliant casting location scenery story direction <UNK> really suited the part they played and you could just imagine being there robert <UNK> is an amazing actor and now the same being director <UNK> father came from the same <UNK> island as myself so i loved the fact there was a real connection with this film the witty <UNK> throughout the film were great it was just brilliant so much that i bought the film as soon as it was released for <UNK> and would recommend it to everyone to watch and the fly <UNK> was amazing really cried at the end it was so sad and you know what they say if you cry at a film it must have been good and this definitely was also <UNK> to the two little <UNK> that played the <UNK> of norman and paul they were just brilliant children are often left out of the <UNK> list i think because the stars that play them all grown up are such a big <UNK> for the whole film but these children are amazing and should be <UNK> for what they have done don't you think the whole story was so lovely because it was true and was <UNK> life after all that was <UNK> with us all\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Oxp3l0TJzgq"
      },
      "source": [
        "## 각 데이터의 길이"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sx0jFfDqHh4q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a9c9e46-d403-4f50-dd33-c491d728666e"
      },
      "source": [
        "print(len(train_x[0]))\n",
        "print(len(train_x[1]))\n",
        "print(len(train_x[2]))\n",
        "print(len(train_x[3]))\n",
        "print(len(train_x[4]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "218\n",
            "189\n",
            "141\n",
            "550\n",
            "147\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICmUFp5tNmGp"
      },
      "source": [
        "## 데이터 길이 일정하게 하기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QpkgDH3CO4gG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d09846cf-112b-40ca-bce6-95410866212d"
      },
      "source": [
        "print(train_x[0])\n",
        "print(len(train_x[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65, 458, 2, 66, 3941, 4, 173, 36, 256, 5, 25, 100, 43, 838, 112, 50, 670, 2, 9, 35, 480, 284, 5, 150, 4, 172, 112, 167, 2, 336, 385, 39, 4, 172, 2, 1111, 17, 546, 38, 13, 447, 4, 192, 50, 16, 6, 147, 2025, 19, 14, 22, 4, 1920, 2, 469, 4, 22, 71, 87, 12, 16, 43, 530, 38, 76, 15, 13, 1247, 4, 22, 17, 515, 17, 12, 16, 626, 18, 2, 5, 62, 386, 12, 8, 316, 8, 106, 5, 4, 2223, 2, 16, 480, 66, 3785, 33, 4, 130, 12, 16, 38, 619, 5, 25, 124, 51, 36, 135, 48, 25, 1415, 33, 6, 22, 12, 215, 28, 77, 52, 5, 14, 407, 16, 82, 2, 8, 4, 107, 117, 2, 15, 256, 4, 2, 7, 3766, 5, 723, 36, 71, 43, 530, 476, 26, 400, 317, 46, 7, 4, 2, 1029, 13, 104, 88, 4, 381, 15, 297, 98, 32, 2071, 56, 26, 141, 6, 194, 2, 18, 4, 226, 22, 21, 134, 476, 26, 480, 5, 144, 30, 2, 18, 51, 36, 28, 224, 92, 25, 104, 4, 226, 65, 16, 38, 1334, 88, 12, 16, 283, 5, 16, 2, 113, 103, 32, 15, 16, 2, 19, 178, 32]\n",
            "218\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0C6fLRb_HA8A",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f03d31e2-4dcd-42c7-f58e-bd4f348cf2bb"
      },
      "source": [
        "from tensorflow.keras.preprocessing import sequence\n",
        "\n",
        "train_x = sequence.pad_sequences(train_x, maxlen=400, padding='post')\n",
        "test_x = sequence.pad_sequences(test_x, maxlen=400, padding='post')\n",
        "print(train_x.shape)\n",
        "print(test_x.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25000, 400)\n",
            "(25000, 400)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K6gb4O2OeWcO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a3eda8f5-b04e-4daa-fd13-04feda8f4d39"
      },
      "source": [
        "print(train_x[0])\n",
        "print(len(train_x[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[   1   14   22   16   43  530  973 1622 1385   65  458    2   66 3941\n",
            "    4  173   36  256    5   25  100   43  838  112   50  670    2    9\n",
            "   35  480  284    5  150    4  172  112  167    2  336  385   39    4\n",
            "  172    2 1111   17  546   38   13  447    4  192   50   16    6  147\n",
            " 2025   19   14   22    4 1920    2  469    4   22   71   87   12   16\n",
            "   43  530   38   76   15   13 1247    4   22   17  515   17   12   16\n",
            "  626   18    2    5   62  386   12    8  316    8  106    5    4 2223\n",
            "    2   16  480   66 3785   33    4  130   12   16   38  619    5   25\n",
            "  124   51   36  135   48   25 1415   33    6   22   12  215   28   77\n",
            "   52    5   14  407   16   82    2    8    4  107  117    2   15  256\n",
            "    4    2    7 3766    5  723   36   71   43  530  476   26  400  317\n",
            "   46    7    4    2 1029   13  104   88    4  381   15  297   98   32\n",
            " 2071   56   26  141    6  194    2   18    4  226   22   21  134  476\n",
            "   26  480    5  144   30    2   18   51   36   28  224   92   25  104\n",
            "    4  226   65   16   38 1334   88   12   16  283    5   16    2  113\n",
            "  103   32   15   16    2   19  178   32    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0]\n",
            "400\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "weGKXhpbNsm7"
      },
      "source": [
        "# embedding 학습하며 예측 실행"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yoibr3xQN0Vq"
      },
      "source": [
        "## CNN 모델 사용"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYlvpK2neGdv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 846
        },
        "outputId": "03cd6f91-eebd-44e8-83e3-e454e2fa3d95"
      },
      "source": [
        "from tensorflow.keras.preprocessing import sequence\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout, Activation\n",
        "from tensorflow.keras.layers import Embedding\n",
        "from tensorflow.keras.layers import Conv1D, GlobalMaxPooling1D\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Input(400))\n",
        "model.add(Embedding(VOCA_SIZE, EMBEDDING_SIZE)) # 텍스트는 임베딩 해서 사용한다.\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Conv1D(250, 3))\n",
        "model.add(GlobalMaxPooling1D())\n",
        "model.add(Dense(250))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(1))\n",
        "model.add(Activation('sigmoid'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "model.fit(train_x, train_y, batch_size=32, epochs=10, validation_data=(test_x, test_y))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 400, 64)           256000    \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 400, 64)           0         \n",
            "_________________________________________________________________\n",
            "conv1d (Conv1D)              (None, 398, 250)          48250     \n",
            "_________________________________________________________________\n",
            "global_max_pooling1d (Global (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 250)               62750     \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "activation (Activation)      (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 1)                 251       \n",
            "_________________________________________________________________\n",
            "activation_1 (Activation)    (None, 1)                 0         \n",
            "=================================================================\n",
            "Total params: 367,251\n",
            "Trainable params: 367,251\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-348e68f5bcad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'binary_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'adam'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1182\u001b[0m                 _r=1):\n\u001b[1;32m   1183\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1184\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1185\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1186\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    883\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    884\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    886\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    931\u001b[0m       \u001b[0;31m# This is the first call of __call__, so we have to initialize.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    932\u001b[0m       \u001b[0minitializers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 933\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_initializers_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    934\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    935\u001b[0m       \u001b[0;31m# At this point we know that the initialization is complete (or less\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_initialize\u001b[0;34m(self, args, kwds, add_initializers_to)\u001b[0m\n\u001b[1;32m    758\u001b[0m     self._concrete_stateful_fn = (\n\u001b[1;32m    759\u001b[0m         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access\n\u001b[0;32m--> 760\u001b[0;31m             *args, **kwds))\n\u001b[0m\u001b[1;32m    761\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    762\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minvalid_creator_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0munused_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0munused_kwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_get_concrete_function_internal_garbage_collected\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3064\u001b[0m       \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3065\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3066\u001b[0;31m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3067\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3068\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_maybe_define_function\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   3461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3462\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmissed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcall_context_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3463\u001b[0;31m           \u001b[0mgraph_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_graph_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3464\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprimary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_create_graph_function\u001b[0;34m(self, args, kwargs, override_flat_arg_shapes)\u001b[0m\n\u001b[1;32m   3306\u001b[0m             \u001b[0marg_names\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0marg_names\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3307\u001b[0m             \u001b[0moverride_flat_arg_shapes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverride_flat_arg_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3308\u001b[0;31m             capture_by_value=self._capture_by_value),\n\u001b[0m\u001b[1;32m   3309\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_attributes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3310\u001b[0m         \u001b[0mfunction_spec\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction_spec\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36mfunc_graph_from_py_func\u001b[0;34m(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes, acd_record_initial_resource_uses)\u001b[0m\n\u001b[1;32m   1005\u001b[0m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moriginal_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munwrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpython_func\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1007\u001b[0;31m       \u001b[0mfunc_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpython_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mfunc_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfunc_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1008\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1009\u001b[0m       \u001b[0;31m# invariant: `func_outputs` contains only Tensors, CompositeTensors,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36mwrapped_fn\u001b[0;34m(*args, **kwds)\u001b[0m\n\u001b[1;32m    666\u001b[0m         \u001b[0;31m# the function a weak reference to itself to avoid a reference cycle.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompile_with_xla\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 668\u001b[0;31m           \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mweak_wrapped_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__wrapped__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    669\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    988\u001b[0m                     \u001b[0mrecursive\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    989\u001b[0m                     \u001b[0moptional_features\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mautograph_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 990\u001b[0;31m                     \u001b[0muser_requested\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    991\u001b[0m                 ))\n\u001b[1;32m    992\u001b[0m           \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint:disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mconverted_call\u001b[0;34m(f, args, kwargs, caller_fn_scope, options)\u001b[0m\n\u001b[1;32m    442\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 444\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconverted_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0meffective_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    445\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    446\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconverted_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0meffective_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtf__train_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m     13\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m                     \u001b[0mdo_return\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m                     \u001b[0mretval_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconverted_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mag__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m                 \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m                     \u001b[0mdo_return\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mconverted_call\u001b[0;34m(f, args, kwargs, caller_fn_scope, options)\u001b[0m\n\u001b[1;32m    380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_requested\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconversion\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_allowlisted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_call_unconverted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m   \u001b[0;31m# internal_convert_user_code is for example turned off when issuing a dynamic\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36m_call_unconverted\u001b[0;34m(f, args, kwargs, options, update_cache)\u001b[0m\n\u001b[1;32m    462\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 464\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    466\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mstep_function\u001b[0;34m(model, iterator)\u001b[0m\n\u001b[1;32m    840\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    841\u001b[0m       \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 842\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistribute_strategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    843\u001b[0m       outputs = reduce_per_replica(\n\u001b[1;32m    844\u001b[0m           outputs, self.distribute_strategy, reduction='first')\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m   1284\u001b[0m       fn = autograph.tf_convert(\n\u001b[1;32m   1285\u001b[0m           fn, autograph_ctx.control_status_ctx(), convert_by_default=False)\n\u001b[0;32m-> 1286\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mcall_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   2847\u001b[0m       \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2848\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2849\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2850\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2851\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m_call_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   3630\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3631\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mReplicaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplica_id_in_sync_group\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3632\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3634\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    690\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mconversion_ctx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 692\u001b[0;31m           \u001b[0;32mreturn\u001b[0m \u001b[0mconverted_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    693\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint:disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    694\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ag_error_metadata'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mconverted_call\u001b[0;34m(f, args, kwargs, caller_fn_scope, options)\u001b[0m\n\u001b[1;32m    380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_requested\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconversion\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_allowlisted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_call_unconverted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m   \u001b[0;31m# internal_convert_user_code is for example turned off when issuing a dynamic\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36m_call_unconverted\u001b[0;34m(f, args, kwargs, options, update_cache)\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 463\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    464\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mrun_step\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    833\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    834\u001b[0m       \u001b[0;32mdef\u001b[0m \u001b[0mrun_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 835\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    836\u001b[0m         \u001b[0;31m# Ensure counter is updated only if `train_step` succeeds.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    837\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol_dependencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_minimum_control_deps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_step\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    789\u001b[0m           y, y_pred, sample_weight, regularization_losses=self.losses)\n\u001b[1;32m    790\u001b[0m     \u001b[0;31m# Run backwards pass.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 791\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainable_variables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    792\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompiled_metrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    793\u001b[0m     \u001b[0;31m# Collect metrics to return\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py\u001b[0m in \u001b[0;36mminimize\u001b[0;34m(self, loss, var_list, grad_loss, name, tape)\u001b[0m\n\u001b[1;32m    519\u001b[0m     \"\"\"\n\u001b[1;32m    520\u001b[0m     grads_and_vars = self._compute_gradients(\n\u001b[0;32m--> 521\u001b[0;31m         loss, var_list=var_list, grad_loss=grad_loss, tape=tape)\n\u001b[0m\u001b[1;32m    522\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads_and_vars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py\u001b[0m in \u001b[0;36m_compute_gradients\u001b[0;34m(self, loss, var_list, grad_loss, tape)\u001b[0m\n\u001b[1;32m    570\u001b[0m     \u001b[0mvar_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvar_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/gradients\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 572\u001b[0;31m       \u001b[0mgrads_and_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    574\u001b[0m     self._assert_valid_dtypes([\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py\u001b[0m in \u001b[0;36m_get_gradients\u001b[0;34m(self, tape, loss, var_list, grad_loss)\u001b[0m\n\u001b[1;32m    452\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_get_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m     \u001b[0;34m\"\"\"Called in `minimize` to compute gradients from loss.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m     \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1088\u001b[0m         \u001b[0moutput_gradients\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1089\u001b[0m         \u001b[0msources_raw\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_sources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1090\u001b[0;31m         unconnected_gradients=unconnected_gradients)\n\u001b[0m\u001b[1;32m   1091\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1092\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/imperative_grad.py\u001b[0m in \u001b[0;36mimperative_grad\u001b[0;34m(tape, target, sources, output_gradients, sources_raw, unconnected_gradients)\u001b[0m\n\u001b[1;32m     75\u001b[0m       \u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m       \u001b[0msources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m       compat.as_str(unconnected_gradients.value))\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36m_gradient_function\u001b[0;34m(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads, skip_input_indices, forward_pass_name_scope)\u001b[0m\n\u001b[1;32m    157\u001b[0m       \u001b[0mgradient_name_scope\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mforward_pass_name_scope\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient_name_scope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 159\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    160\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_grad.py\u001b[0m in \u001b[0;36m_MeanGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m    254\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_MeanGrad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m   \u001b[0;34m\"\"\"Gradient for Mean.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m   \u001b[0msum_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_SumGrad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m   \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m   \u001b[0moutput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_grad.py\u001b[0m in \u001b[0;36m_SumGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m    214\u001b[0m       \u001b[0;31m# more sense.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m       output_shape_kept_dims = math_ops.reduced_shape(input_shape,\n\u001b[0;32m--> 216\u001b[0;31m                                                       op.inputs[1])\n\u001b[0m\u001b[1;32m    217\u001b[0m     \u001b[0mgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_shape_kept_dims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mreduced_shape\u001b[0;34m(input_shape, axes)\u001b[0m\n\u001b[1;32m   4454\u001b[0m       [\n\u001b[1;32m   4455\u001b[0m           \u001b[0minput_shape\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# [2, 3, 5, 7]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4456\u001b[0;31m           \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxes_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4457\u001b[0m       ])  # [1, 1]\n\u001b[1;32m   4458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mones\u001b[0;34m(shape, dtype, name)\u001b[0m\n\u001b[1;32m   3226\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3227\u001b[0m       \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Ensure it's a vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3228\u001b[0;31m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3229\u001b[0m   \u001b[0;32massert\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase_dtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3230\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py\u001b[0m in \u001b[0;36mconstant\u001b[0;34m(value, dtype, shape, name)\u001b[0m\n\u001b[1;32m    270\u001b[0m   \"\"\"\n\u001b[1;32m    271\u001b[0m   return _constant_impl(value, dtype, shape, name, verify_shape=False,\n\u001b[0;32m--> 272\u001b[0;31m                         allow_broadcast=True)\n\u001b[0m\u001b[1;32m    273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py\u001b[0m in \u001b[0;36m_constant_impl\u001b[0;34m(value, dtype, shape, name, verify_shape, allow_broadcast)\u001b[0m\n\u001b[1;32m    292\u001b[0m   \u001b[0mattrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"value\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtensor_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"dtype\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdtype_value\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m   const_tensor = g._create_op_internal(  # pylint: disable=protected-access\n\u001b[0;32m--> 294\u001b[0;31m       \"Const\", [], [dtype_value.type], attrs=attrs, name=name).outputs[0]\n\u001b[0m\u001b[1;32m    295\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mop_callbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_invoke_op_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[1;32m    599\u001b[0m     return super(FuncGraph, self)._create_op_internal(  # pylint: disable=protected-access\n\u001b[1;32m    600\u001b[0m         \u001b[0mop_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_def\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 601\u001b[0;31m         compute_device)\n\u001b[0m\u001b[1;32m    602\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    603\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcapture\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[1;32m   3567\u001b[0m           \u001b[0minput_types\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3568\u001b[0m           \u001b[0moriginal_op\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_default_original_op\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3569\u001b[0;31m           op_def=op_def)\n\u001b[0m\u001b[1;32m   3570\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_op_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompute_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompute_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3571\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)\u001b[0m\n\u001b[1;32m   2055\u001b[0m       \u001b[0mtf_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc_api_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2056\u001b[0m       \u001b[0moutput_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpywrap_tf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_OperationOutputType\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2057\u001b[0;31m       \u001b[0mtensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_with_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf_output\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2058\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_outputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2059\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_create_with_tf_output\u001b[0;34m(op, value_index, dtype, tf_output)\u001b[0m\n\u001b[1;32m    403\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_create_with_tf_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m     \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m     \u001b[0mret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tf_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, op, value_index, dtype)\u001b[0m\n\u001b[1;32m    387\u001b[0m     \u001b[0;31m# to easily navigate a computation graph.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    388\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_consumers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 389\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0muid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    390\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36muid\u001b[0;34m()\u001b[0m\n\u001b[1;32m    219\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0muid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m   \u001b[0;34m\"\"\"A unique (within this program execution) integer.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mpywrap_tfe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTFE_Py_UID\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28CiZH01d5RR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7ac40096-684f-4130-920a-3f7a7948a713"
      },
      "source": [
        "loss, acc = model.evaluate(test_x, test_y)\n",
        "print(\"loss =\", loss)\n",
        "print(\"acc =\", acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25000/25000 [==============================] - 1s 58us/sample - loss: 0.5359 - acc: 0.8866\n",
            "loss = 0.5358699916682392\n",
            "acc = 0.88656\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AQnEiGI_Y-2o"
      },
      "source": [
        "## RNN 모델 사용\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HKWcjknZZAiF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c1c04343-a35b-44f4-a73c-f9aaf365078b"
      },
      "source": [
        "from tensorflow.keras.preprocessing import sequence\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout, Activation\n",
        "from tensorflow.keras.layers import Embedding\n",
        "from tensorflow.keras.layers import Conv1D, GlobalMaxPooling1D\n",
        "from tensorflow.keras.layers import Bidirectional, LSTM, LSTM\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Input(400))\n",
        "model.add(Embedding(VOCA_SIZE, EMBEDDING_SIZE))\n",
        "model.add(Dropout(0.2))\n",
        "# model.add(Conv1D(250, 3))\n",
        "# model.add(GlobalMaxPooling1D())\n",
        "model.add(Bidirectional(LSTM(64))) # ADD\n",
        "model.add(Dense(250))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(1))\n",
        "model.add(Activation('sigmoid'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "model.fit(train_x, train_y, batch_size=32, epochs=10, validation_data=(test_x, test_y))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 400, 64)           256000    \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 400, 64)           0         \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 128)               66560     \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 250)               32250     \n",
            "_________________________________________________________________\n",
            "dropout_3 (Dropout)          (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "activation_2 (Activation)    (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 251       \n",
            "_________________________________________________________________\n",
            "activation_3 (Activation)    (None, 1)                 0         \n",
            "=================================================================\n",
            "Total params: 355,061\n",
            "Trainable params: 355,061\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Train on 25000 samples, validate on 25000 samples\n",
            "Epoch 1/10\n",
            "25000/25000 [==============================] - 38s 2ms/sample - loss: 0.5963 - acc: 0.6566 - val_loss: 0.6832 - val_acc: 0.5032\n",
            "Epoch 2/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.4824 - acc: 0.7720 - val_loss: 0.5637 - val_acc: 0.6978\n",
            "Epoch 3/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.4444 - acc: 0.8060 - val_loss: 0.4524 - val_acc: 0.8064\n",
            "Epoch 4/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.5160 - acc: 0.7426 - val_loss: 0.5487 - val_acc: 0.7120\n",
            "Epoch 5/10\n",
            "25000/25000 [==============================] - 31s 1ms/sample - loss: 0.5243 - acc: 0.7465 - val_loss: 0.5407 - val_acc: 0.7336\n",
            "Epoch 6/10\n",
            "25000/25000 [==============================] - 31s 1ms/sample - loss: 0.4143 - acc: 0.8188 - val_loss: 0.3412 - val_acc: 0.8575\n",
            "Epoch 7/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.2737 - acc: 0.8944 - val_loss: 0.3177 - val_acc: 0.8665\n",
            "Epoch 8/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.2300 - acc: 0.9128 - val_loss: 0.3223 - val_acc: 0.8743\n",
            "Epoch 9/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.2046 - acc: 0.9237 - val_loss: 0.3032 - val_acc: 0.8779\n",
            "Epoch 10/10\n",
            "25000/25000 [==============================] - 30s 1ms/sample - loss: 0.1825 - acc: 0.9354 - val_loss: 0.3039 - val_acc: 0.8767\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7ff73357ed30>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nd53lZYFZY0U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e5ae600-a813-4320-849b-deded9a36085"
      },
      "source": [
        "loss, acc = model.evaluate(test_x, test_y)\n",
        "print(\"loss =\", loss)\n",
        "print(\"acc =\", acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25000/25000 [==============================] - 10s 381us/sample - loss: 0.3039 - acc: 0.8767\n",
            "loss = 0.303929480047226\n",
            "acc = 0.87668\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gQyQSM1UVCh7"
      },
      "source": [
        "# 기존 Word2Vec 사용하여 예측 실행\n",
        "\n",
        "새로 Embedding을 학습하지 않고 이미 학습된 Word2Vec을 사용."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FrFo9E4cYDmR",
        "outputId": "38fbeeb2-73f1-4485-f313-c3bbbf8f40d3"
      },
      "source": [
        "!pip install gensim"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gensim in /usr/local/lib/python3.7/dist-packages (3.6.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.19.5)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.15.0)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (5.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WKDrAvegcSAN"
      },
      "source": [
        "아래 코드로 로딩이 가능한데, 데이터가 엄청 크다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T75NMTTjZYm9"
      },
      "source": [
        "# import gensim.downloader as api\n",
        "# word2vec = api.load('word2vec-google-news-300')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2TUWPXMEcVZF"
      },
      "source": [
        "대신 슬림한 것을 사용한다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QNTK-z2tbsK9",
        "outputId": "e017b2e2-01bf-4977-b235-252752343e93"
      },
      "source": [
        "!wget https://github.com/eyaler/word2vec-slim/raw/master/GoogleNews-vectors-negative300-SLIM.bin.gz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-08-27 01:10:11--  https://github.com/eyaler/word2vec-slim/raw/master/GoogleNews-vectors-negative300-SLIM.bin.gz\n",
            "Resolving github.com (github.com)... 192.30.255.113\n",
            "Connecting to github.com (github.com)|192.30.255.113|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://media.githubusercontent.com/media/eyaler/word2vec-slim/master/GoogleNews-vectors-negative300-SLIM.bin.gz [following]\n",
            "--2021-08-27 01:10:11--  https://media.githubusercontent.com/media/eyaler/word2vec-slim/master/GoogleNews-vectors-negative300-SLIM.bin.gz\n",
            "Resolving media.githubusercontent.com (media.githubusercontent.com)... 185.199.110.133, 185.199.108.133, 185.199.111.133, ...\n",
            "Connecting to media.githubusercontent.com (media.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 276467217 (264M) [application/octet-stream]\n",
            "Saving to: ‘GoogleNews-vectors-negative300-SLIM.bin.gz’\n",
            "\n",
            "GoogleNews-vectors- 100%[===================>] 263.66M   136MB/s    in 1.9s    \n",
            "\n",
            "2021-08-27 01:10:13 (136 MB/s) - ‘GoogleNews-vectors-negative300-SLIM.bin.gz’ saved [276467217/276467217]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_PRDWT5b9fi"
      },
      "source": [
        "!gzip -d GoogleNews-vectors-negative300-SLIM.bin.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JtCskE5bzYM"
      },
      "source": [
        "from gensim.models import KeyedVectors\n",
        "\n",
        "word2vec = KeyedVectors.load_word2vec_format(\"GoogleNews-vectors-negative300-SLIM.bin\", binary=True)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eWIBrUKmciet",
        "outputId": "16dd485b-c635-4685-afb9-9474fb4e5be7"
      },
      "source": [
        "print(len(word2vec['tiger']))\n",
        "print(word2vec['tiger'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "300\n",
            "[-1.95743926e-02  5.22915944e-02 -5.08934222e-02  4.75378111e-02\n",
            "  4.19451296e-02  7.27048889e-03  1.43312523e-02 -1.00668306e-02\n",
            "  7.27048889e-02  6.37915509e-04 -9.33934498e-05 -5.87231815e-02\n",
            " -8.55680630e-02 -1.06260993e-01 -1.27513185e-01 -5.95620833e-02\n",
            " -5.42490333e-02 -1.63848163e-04 -7.01881796e-02 -5.98417185e-02\n",
            "  7.21456185e-02 -1.06820263e-01  5.48083000e-02 -5.94222639e-03\n",
            " -8.50087926e-02  3.55135426e-02 -1.31987333e-01  4.08265926e-02\n",
            "  2.04692230e-01  3.39755528e-02 -4.22247648e-02 -3.29968333e-02\n",
            " -2.58661620e-02 -2.43281741e-02 -9.00422111e-02  4.41822037e-02\n",
            " -5.64861074e-02  8.27717185e-02  9.33978185e-02  7.99753815e-02\n",
            " -6.11699792e-03 -2.22309176e-02  3.71913463e-02  5.14526926e-02\n",
            "  8.16531852e-02 -7.01881796e-02 -2.06929296e-02 -3.10393963e-02\n",
            "  1.13531485e-01  7.88568407e-02 -8.44495296e-02  8.22124556e-02\n",
            " -2.67400197e-03 -6.15195222e-02  2.96412241e-02 -3.80302519e-02\n",
            "  5.45286685e-02 -7.27048889e-02  1.09616600e-01  1.02765569e-02\n",
            "  2.88023222e-02  2.99208593e-02  4.72581796e-02 -6.29176944e-02\n",
            "  3.94284204e-02 -5.39693981e-02  2.01336611e-02 -2.57263463e-02\n",
            " -6.20787926e-02 -2.15842645e-03 -2.48874426e-02  1.70576852e-02\n",
            "  2.01336611e-02 -1.68479607e-02 -1.33525329e-02 -1.04163736e-02\n",
            "  3.29968333e-02 -9.73126963e-02 -2.49923067e-03 -5.52714453e-04\n",
            " -5.38295833e-03 -5.45286685e-02  4.69785444e-02  7.27048889e-02\n",
            " -1.00109041e-01 -2.97810417e-02 -2.09725648e-02 -6.01213500e-02\n",
            " -2.25105528e-02 -3.36959213e-02 -1.18844528e-02  1.24157585e-01\n",
            " -1.87354907e-02  1.68479607e-02 -8.55680630e-02 -1.21920511e-01\n",
            "  5.28508611e-02  3.45348231e-02  5.17323241e-02 -7.27048889e-02\n",
            " -2.19512843e-02 -4.41822037e-02 -7.37535162e-03 -4.75378111e-02\n",
            "  6.51547685e-02 -8.44495296e-02 -2.76837852e-02 -3.48144583e-02\n",
            "  3.28570195e-02 -3.88691537e-02 -1.48765385e-01  2.40485407e-02\n",
            "  8.45893472e-03  8.27717185e-02 -5.62064722e-02 -2.47476269e-02\n",
            " -4.13858593e-02 -1.39257833e-01 -7.88568407e-02 -8.38902593e-03\n",
            " -5.55772940e-03  5.56472056e-02 -1.04583189e-01 -3.65272164e-03\n",
            " -4.11062278e-02  1.78965889e-02  1.09616600e-01 -2.41883583e-02\n",
            "  4.30636667e-02 -1.06260993e-01 -4.08265926e-02  3.48144583e-02\n",
            " -9.28385556e-02  4.27840315e-02 -6.48751333e-02  5.36897667e-02\n",
            "  2.62856148e-02  6.81608357e-03  2.60059796e-02  1.69877764e-02\n",
            "  5.36897667e-02 -8.83644074e-02 -2.12521981e-02  1.90151259e-02\n",
            "  1.37719838e-02  8.72458667e-02 -5.00545204e-02  8.10939148e-02\n",
            "  4.31685289e-03 -2.99208593e-02  1.33105874e-01  4.39025685e-02\n",
            " -7.88568407e-02  5.84435463e-02  5.48083000e-02 -7.15863556e-02\n",
            "  3.46047315e-03 -1.01367394e-02  4.19451296e-02 -5.17323241e-02\n",
            "  3.57931778e-02 -5.90028130e-02  6.06806204e-02 -1.69877764e-02\n",
            "  2.48175347e-03 -2.33494546e-02  7.71790370e-02 -7.21456185e-02\n",
            " -7.79480301e-03  3.57931778e-02 -2.65652481e-02 -2.05531139e-02\n",
            "  1.27233556e-02  3.71913463e-02  2.01336611e-02 -2.39087231e-02\n",
            "  1.06260991e-02  2.71245167e-02 -5.73250093e-02 -9.33978185e-02\n",
            " -7.88568407e-02 -6.82307407e-02 -3.50940898e-02 -3.77506167e-02\n",
            "  1.62886921e-02 -6.68325722e-02  2.98859039e-03  5.34101315e-02\n",
            "  6.68325722e-02  1.56595148e-02 -1.92947593e-02 -1.24716848e-01\n",
            " -7.43826926e-02  5.14526926e-02  3.08995787e-02  1.18844528e-02\n",
            "  5.59268380e-03 -7.49419630e-02 -4.19451296e-02  1.71275940e-02\n",
            "  3.66320796e-02 -4.19451296e-02  1.28456962e-03 -4.25043963e-02\n",
            "  3.27172019e-02  6.68500492e-04  1.01227574e-01 -6.51547685e-02\n",
            "  3.22977491e-02 -1.17446361e-02 -1.52121007e-01 -2.08327472e-02\n",
            "  3.94284204e-02 -1.76728815e-01 -3.48144583e-02  8.44495296e-02\n",
            "  1.09057337e-01 -1.11154588e-02  1.71975028e-02  6.73918426e-02\n",
            "  3.80302519e-02  8.17930046e-03 -7.55012333e-02 -3.97080556e-02\n",
            "  4.61396426e-02 -3.07597611e-02 -3.02004926e-02  3.36959213e-02\n",
            "  8.83644074e-02 -1.04862824e-02  2.46078093e-02 -4.27840315e-02\n",
            "  4.66989093e-02  5.43975912e-04 -3.74709815e-02 -3.85895185e-02\n",
            " -1.64285097e-02  2.11123824e-02 -3.17384824e-02 -3.52339074e-02\n",
            "  7.71790370e-02 -8.10939148e-02 -2.41883583e-02 -6.93492815e-02\n",
            "  4.39025685e-02  2.83129630e-03  9.15801991e-03 -5.95620833e-02\n",
            "  8.61273333e-02 -5.31304954e-03 -7.77383074e-02  5.53675704e-02\n",
            " -2.82430537e-02  3.57931778e-02 -4.71882708e-03  1.78965889e-02\n",
            "  2.42932211e-03 -5.17323241e-02 -6.12398870e-02 -4.75378111e-02\n",
            " -7.94161111e-02  3.97080556e-02  1.61069289e-01  1.07659167e-02\n",
            "  6.29176944e-02  6.12398870e-02 -1.39257833e-01 -2.82430537e-02\n",
            " -8.94829445e-03 -5.67657426e-02  8.69487587e-04  1.27233556e-02\n",
            "  1.15349106e-02  1.00668311e-01  7.15863556e-02  2.86625046e-02\n",
            "  5.54899103e-04  1.53798806e-02 -5.84435463e-02  1.45934091e-03\n",
            "  7.55012333e-02 -2.54467111e-02 -5.87231815e-02 -3.07597611e-02\n",
            "  3.66320796e-02 -2.60059796e-02 -3.04801278e-02  3.86244734e-03\n",
            "  8.94829407e-02 -6.64131204e-03 -3.02004926e-02 -1.44710699e-02\n",
            "  4.19451296e-02 -7.51516875e-03  2.58661620e-02  1.13251852e-02\n",
            "  8.42398033e-03  5.45286685e-02 -2.92217731e-02  4.94952537e-02]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z_0GYR8jcb1g",
        "outputId": "d6e8c9e6-3757-465b-c411-a59a335d7fd9"
      },
      "source": [
        "word2vec.most_similar('samsung')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('htc', 0.7326200604438782),\n",
              " ('nokia', 0.7076619267463684),\n",
              " ('sony', 0.6818499565124512),\n",
              " ('motorola', 0.6689046025276184),\n",
              " ('ipad', 0.6625324487686157),\n",
              " ('iphone', 0.6582240462303162),\n",
              " ('tmo', 0.6467313766479492),\n",
              " ('treo', 0.6441471576690674),\n",
              " ('macbook', 0.6297854781150818),\n",
              " ('fone', 0.6272913217544556)]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WovKk-gvWCKK",
        "outputId": "f2db9a43-17e1-4342-8f2f-a3f8195f89f3"
      },
      "source": [
        "word_set = set()\n",
        "\n",
        "for i in range(len(train_x)):\n",
        "  words = decode_review(train_x[i]).split(\" \")\n",
        "  word_set.update(words)\n",
        "\n",
        "for i in range(len(test_x)):\n",
        "  words = decode_review(test_x[i]).split(\" \")\n",
        "  word_set.update(words)\n",
        "\n",
        "print(len(word_set))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3999\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UGBtwK9Mc6fj",
        "outputId": "1eaa5895-f454-4bf9-c21f-5e5be1aa065e"
      },
      "source": [
        "EMBEDDING_SIZE = len(word2vec['tiger'])\n",
        "print(EMBEDDING_SIZE)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "300\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RnqbiMxuVJEu",
        "outputId": "78198e0b-ae67-41aa-d990-31e6b9675e03"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "VOCA_SIZE = 4000 # 어휘 사전의 크기\n",
        "EMBEDDING_SIZE = len(word2vec['tiger'])\n",
        "# EMBEDDING_SIZE = 300 # 단어를 임베딩한 벡터 크기\n",
        "\n",
        "embedding_matrix = np.zeros((VOCA_SIZE, EMBEDDING_SIZE))\n",
        "\n",
        "# tokenizer에 있는 단어 사전을 순회하면서 word2vec의 300차원 vector를 가져옵니다\n",
        "for word in word_set:\n",
        "  if word not in word2vec : continue\n",
        "  idx = word_index[word]\n",
        "  # word, idx\n",
        "  # it 12\n",
        "  # took 562\n",
        "  # kill 516\n",
        "  # rough 2683\n",
        "  embedding_matrix[idx] = word2vec[word]\n",
        "  # embedding_matrix[12] = [ 0.05521541 -0.00023065  0.0347889 ... ]\n",
        "        \n",
        "print(embedding_matrix.shape)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(4000, 300)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ValNpKi1Xwzw",
        "outputId": "9643ec5e-e04c-4969-eae8-2c937a6c472f"
      },
      "source": [
        "from tensorflow.keras.preprocessing import sequence\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout, Activation\n",
        "from tensorflow.keras.layers import Embedding\n",
        "from tensorflow.keras.layers import Conv1D, GlobalMaxPooling1D\n",
        "from tensorflow.keras.layers import Bidirectional, LSTM, LSTM\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Input(400))\n",
        "# model.add(Embedding(VOCA_SIZE, EMBEDDING_SIZE))\n",
        "model.add(Embedding(VOCA_SIZE, \n",
        "                    EMBEDDING_SIZE, \n",
        "                    input_length=400, \n",
        "                    weights=[embedding_matrix],\n",
        "                    trainable=False\n",
        "                   )\n",
        "         )\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Bidirectional(LSTM(64)))\n",
        "model.add(Dense(250))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(1))\n",
        "model.add(Activation('sigmoid'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(train_x, train_y, batch_size=32, epochs=10, validation_data=(test_x, test_y))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 400, 300)          1200000   \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 400, 300)          0         \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 128)               186880    \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 250)               32250     \n",
            "_________________________________________________________________\n",
            "dropout_3 (Dropout)          (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "activation_2 (Activation)    (None, 250)               0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 251       \n",
            "_________________________________________________________________\n",
            "activation_3 (Activation)    (None, 1)                 0         \n",
            "=================================================================\n",
            "Total params: 1,419,381\n",
            "Trainable params: 219,381\n",
            "Non-trainable params: 1,200,000\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "782/782 [==============================] - 39s 43ms/step - loss: 0.6787 - accuracy: 0.5640 - val_loss: 0.6721 - val_accuracy: 0.5850\n",
            "Epoch 2/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.6704 - accuracy: 0.5833 - val_loss: 0.6767 - val_accuracy: 0.5819\n",
            "Epoch 3/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.6427 - accuracy: 0.6284 - val_loss: 0.5865 - val_accuracy: 0.6974\n",
            "Epoch 4/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.6326 - accuracy: 0.6276 - val_loss: 0.6872 - val_accuracy: 0.5127\n",
            "Epoch 5/10\n",
            "782/782 [==============================] - 32s 42ms/step - loss: 0.6266 - accuracy: 0.6315 - val_loss: 0.5365 - val_accuracy: 0.7265\n",
            "Epoch 6/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.5325 - accuracy: 0.7354 - val_loss: 0.5576 - val_accuracy: 0.7064\n",
            "Epoch 7/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.4985 - accuracy: 0.7550 - val_loss: 0.4914 - val_accuracy: 0.7549\n",
            "Epoch 8/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.4713 - accuracy: 0.7714 - val_loss: 0.4661 - val_accuracy: 0.7836\n",
            "Epoch 9/10\n",
            "782/782 [==============================] - 32s 42ms/step - loss: 0.4411 - accuracy: 0.7924 - val_loss: 0.4333 - val_accuracy: 0.7978\n",
            "Epoch 10/10\n",
            "782/782 [==============================] - 32s 41ms/step - loss: 0.4124 - accuracy: 0.8069 - val_loss: 0.4255 - val_accuracy: 0.8008\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f0c2bc62d50>"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bwf4ZgPmdObl",
        "outputId": "69f79aab-ab7c-4788-f10f-f4efe215a550"
      },
      "source": [
        "loss, acc = model.evaluate(test_x, test_y)\n",
        "print(\"loss =\", loss)\n",
        "print(\"acc =\", acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "782/782 [==============================] - 11s 14ms/step - loss: 0.4255 - accuracy: 0.8008\n",
            "loss = 0.4255183935165405\n",
            "acc = 0.8008000254631042\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}