Group28_Homework7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "200c1759a0e64aad994feb4854557b0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3480ee4081dc46519ff9e3fd4e1eaa61",
              "IPY_MODEL_88dd76a4ac7040488fc9cba36c537b7b",
              "IPY_MODEL_073e9a1e248f48d7ae7fc45c9c6c781c"
            ],
            "layout": "IPY_MODEL_d2edf588f77948508e095861b04671c2"
          }
        },
        "3480ee4081dc46519ff9e3fd4e1eaa61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f572fa197ebd4a0a842b7f64e75d62dc",
            "placeholder": "​",
            "style": "IPY_MODEL_ec39e510ec244719b6e0922866e065ca",
            "value": "Dl Completed...: 100%"
          }
        },
        "88dd76a4ac7040488fc9cba36c537b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5330590bc29e46de95092d9b37cf8c10",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4b3d6769a8ea4b8cb252926cf4658f13",
            "value": 5
          }
        },
        "073e9a1e248f48d7ae7fc45c9c6c781c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea9b6170ed374f439ad88987dde6b6c6",
            "placeholder": "​",
            "style": "IPY_MODEL_7fb7671b13f34955b05cafa8265b6d01",
            "value": " 5/5 [00:02&lt;00:00,  2.10 file/s]"
          }
        },
        "d2edf588f77948508e095861b04671c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f572fa197ebd4a0a842b7f64e75d62dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec39e510ec244719b6e0922866e065ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5330590bc29e46de95092d9b37cf8c10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b3d6769a8ea4b8cb252926cf4658f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ea9b6170ed374f439ad88987dde6b6c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7fb7671b13f34955b05cafa8265b6d01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "##Homework 7: Implement LSTM\n",
        "\n",
        "Groupmembers:\n",
        "- Carmen Amme (994813)\n",
        "- Anneke Büürma (995025)\n",
        "- Wilfred Okajevo (998818)\n",
        "\n"
      ],
      "metadata": {
        "id": "GvopVO_TkDCP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2.1 Prepare the Dataset\n",
        "To create the sequences you could simply .batch() twice, once to create the sequences and the second time to create the batches."
      ],
      "metadata": {
        "id": "4rQXhpR0kZdh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load packages\n",
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import datetime\n",
        "\n"
      ],
      "metadata": {
        "id": "E-RjquSZkVFU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finding the targets"
      ],
      "metadata": {
        "id": "RMlo6Js_lCda"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def new_target_fnc(ds, sequence_len):\n",
        "  \"\"\"\n",
        "  Creates list of new targets by alternately adding and subtracting\n",
        "  The first digit is added, the second subtracted, the third added, etc\n",
        "  Parameters\n",
        "  ----------\n",
        "  ds : TensorFlowDataset\n",
        "  original mnist dataset containg images and targets as a tuple.\n",
        "  sequence_len : int\n",
        "  indicates at which point the sum has to reset for the new sequence\n",
        "  Returns\n",
        "  -------\n",
        "  l : list\n",
        "  list containing the new targets\n",
        "  \"\"\"\n",
        "\n",
        "  l = list()\n",
        "  for i, elem in enumerate(ds):\n",
        "    if (i % sequence_len) == 0:\n",
        "      l.append(int(elem[1]))\n",
        "    else:\n",
        "      if (i % 2) == 0:\n",
        "        l.append(int(l[i-1] + elem[1]))\n",
        "      else:\n",
        "        l.append(int(l[i-1] - elem[1]))\n",
        "\n",
        "  return l"
      ],
      "metadata": {
        "id": "5UuYK0jDkNll"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def new_targ(ds, sequence_len):\n",
        "  #range_vals is only used to create the change between positive and negative signs for the summation\n",
        "  range_vals = tf.range(10)\n",
        "  #target_digits is a placeholder\n",
        "  target_digits = tf.random.uniform((10,), minval=0, maxval=10, dtype=tf.dtypes.int32)\n",
        "  #this creates the alternating positve and negative signes by checkign whether the entry index modulo 2 is zero (i.e. even entries are positive, uneven ones negative)\n",
        "  #check the tf.where documentation if the usage is confusing for you!\n",
        "  alternating_target_numbers = tf.where(tf.math.floormod(range_vals,2)==0, target_digits, -target_digits)\n",
        "  print(alternating_target_numbers)\n",
        "  #finally we can compute the cumulative sums!\n",
        "  c_sum = tf.math.cumsum(alternating_target_numbers)\n",
        "  print(c_sum)\n",
        "  return c_sum"
      ],
      "metadata": {
        "id": "IieWVLEPpUa0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "2.1.2 Create the datapipeline"
      ],
      "metadata": {
        "id": "_MRsmgXcrY2N"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "(train_ds, val_ds), ds_info = tfds.load ('mnist', split =['train', 'test'], as_supervised = True, with_info = True)\n",
        "\n",
        "# print(\"ds_info: \\n\", ds_info)\n",
        "# tfds.show_examples(train_ds, ds_info)\n",
        "\n",
        "batch_size = 64\n",
        "sequence_len = 6\n",
        "\n",
        "def preprocess(dataset, batchsize, sequence_len):\n",
        "    '''\n",
        "    :param dataset: the dataset to be prepared for input into the network\n",
        "    :param batchsize: the desired batchsize\n",
        "    :param sequence_len: sequence the dataset (you may think this as a another batch)\n",
        "    :return: sequenced dataset with sequential target\n",
        "    '''\n",
        "\n",
        "    # Step 1 - General Preprocessing\n",
        "\n",
        "    # convert data from uint8 to float32\n",
        "    dataset = dataset.map(lambda img, target: (tf.cast(img, tf.float32), target))\n",
        "\n",
        "    # input normalization, just bringing image values from range [0, 255] to [-1, 1]\n",
        "    dataset = dataset.map(lambda img, target: ((img / 128.) - 1., target))\n",
        "\n",
        "    # The output of that lambda function should be a tuple of two tensors of shapes (num_images, height, width, 1) and (num_images, 1) or (num_images,)\n",
        "\n",
        "    # Step 3 - Sequence Batching & Shuffling & Batching & Prefetching\n",
        "    dataset = dataset.batch(sequence_len)\n",
        "\n",
        "    # change the target\n",
        "\n",
        "    # alternate positive, negative target values\n",
        "    range_vals = tf.range(sequence_len)\n",
        "\n",
        "    dataset = dataset.map(lambda img, target:\n",
        "                          (img, tf.where(tf.math.floormod(range_vals,2)==0, target, -target)))\n",
        "\n",
        "    dataset = dataset.map(lambda img, target:\n",
        "                          (img, (tf.math.cumsum(target))))\n",
        "\n",
        "    # cache\n",
        "    dataset = dataset.cache()\n",
        "\n",
        "    # shuffle, batch, prefetch\n",
        "    dataset = dataset.shuffle(1000)\n",
        "    dataset = dataset.batch(batchsize)\n",
        "    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n",
        "\n",
        "    # The shape of your tensors should be (batch, sequence-length, features).\n",
        "    return dataset\n",
        "\n",
        "def expand_dimension(x, y):\n",
        "    return x, tf.expand_dims(y, axis=-1)\n",
        "\n",
        "\n",
        "train_ds = preprocess(train_ds, batch_size, 6)\n",
        "val_ds = preprocess(val_ds, batch_size, 6)\n",
        "\n",
        "train_ds = train_ds.map(expand_dimension)\n",
        "val_ds = train_ds.map(expand_dimension)"
      ],
      "metadata": {
        "id": "Iy8OwN0prcfI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "200c1759a0e64aad994feb4854557b0e",
            "3480ee4081dc46519ff9e3fd4e1eaa61",
            "88dd76a4ac7040488fc9cba36c537b7b",
            "073e9a1e248f48d7ae7fc45c9c6c781c",
            "d2edf588f77948508e095861b04671c2",
            "f572fa197ebd4a0a842b7f64e75d62dc",
            "ec39e510ec244719b6e0922866e065ca",
            "5330590bc29e46de95092d9b37cf8c10",
            "4b3d6769a8ea4b8cb252926cf4658f13",
            "ea9b6170ed374f439ad88987dde6b6c6",
            "7fb7671b13f34955b05cafa8265b6d01"
          ]
        },
        "outputId": "b09ed529-fb39-48db-c135-7c37c880d051"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset 11.06 MiB (download: 11.06 MiB, generated: 21.00 MiB, total: 32.06 MiB) to ~/tensorflow_datasets/mnist/3.0.1...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Dl Completed...:   0%|          | 0/5 [00:00<?, ? file/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "200c1759a0e64aad994feb4854557b0e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset mnist downloaded and prepared to ~/tensorflow_datasets/mnist/3.0.1. Subsequent calls will reuse this data.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for img, label in train_ds.take(1):\n",
        "\n",
        "  shape_ds = img.shape\n",
        "\n",
        "  print(img.shape, label.shape) \n",
        "  # (bs, num_images, height, width, 1) (bs, num_images, 1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cziZN4s_vV90",
        "outputId": "f07c4379-f7b3-4925-d18e-33b72e14b620"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(64, 6, 28, 28, 1) (64, 6, 1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2.2 The CNN & LSTM Network"
      ],
      "metadata": {
        "id": "EswapbNzrurj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# With extended batch shape [4, 7]:\n",
        "input_shape = (64, 6, 28, 28, 1)\n",
        "x = tf.random.normal(input_shape)\n",
        "y = tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=input_shape[1:])(x)\n",
        "print(y.shape)\n"
      ],
      "metadata": {
        "id": "Q20dhKelxWf4",
        "outputId": "2f5d26e5-40f2-4735-9fb8-2d41368002eb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(64, 6, 26, 26, 32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class BasicCNN_LSTM(tf.keras.Model):\n",
        "    def __init__(self):\n",
        "        super(BasicCNN_LSTM, self).__init__()\n",
        "\n",
        "        self.convlayer1 = tf.keras.layers.Conv2D(filters=48, kernel_size=3, padding='same', activation='relu', \n",
        "                                                batch_input_shape=(batch_size, sequence_len, 28, 28, 1))#, input_shape=(28, 28, 1))\n",
        "        self.convlayer2 = tf.keras.layers.Conv2D(filters=48, kernel_size=3, padding='same', activation='relu', \n",
        "                                                batch_input_shape=(batch_size, sequence_len, 28, 28, 1))#, input_shape=(28, 28, 1))\n",
        "        self.convlayer3 = tf.keras.layers.Conv2D(filters=48, kernel_size=3, padding='same', activation='relu', \n",
        "                                                batch_input_shape=(batch_size, sequence_len, 28, 28, 1))#, input_shape=(28, 28, 1))\n",
        "        self.batchnorm1 = tf.keras.layers.BatchNormalization()\n",
        "\n",
        "        self.global_pool = tf.keras.layers.GlobalAvgPool2D()\n",
        "        self.timedist = tf.keras.layers.TimeDistributed(self.global_pool)\n",
        "\n",
        "        self.rnn = tf.keras.layers.RNN(ourlstm(8), return_sequences=True) \n",
        "        self.batchnorm2 = tf.keras.layers.BatchNormalization()\n",
        "\n",
        "        self.outputlayer = tf.keras.layers.Dense(units=1, activation=None) \n",
        "        \n",
        "    @tf.function # Leon: comment it out when debugging\n",
        "    def call(self, x):\n",
        "        # print(f\"initial shape: {x.shape}\")\n",
        "        x = self.convlayer1(x)\n",
        "        x = self.convlayer2(x)\n",
        "        x = self.convlayer3(x)\n",
        "        # print(f\"shape after cnn: {x.shape}\")\n",
        "        \n",
        "        x = self.batchnorm1(x)\n",
        "        x = self.timedist(x) \n",
        "        # print(f\"shape after timedist&pooling: {x.shape}\") # shape should be (bs, sequence-length, features) before LSTM\n",
        "\n",
        "        x = self.rnn(x)\n",
        "        # print(f\"shape after rnn: {x.shape}\")\n",
        "        \n",
        "        x = self.batchnorm2(x)\n",
        "        x = self.outputlayer(x)\n",
        "        # print(f\"shape after output: {x.shape}\")\n",
        "\n",
        "        return x"
      ],
      "metadata": {
        "id": "F9LOg3JXryOI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2.3 LSTM AbstractRNNCell layer"
      ],
      "metadata": {
        "id": "lPwJKRIjvNVu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ourlstm(tf.keras.layers.AbstractRNNCell):\n",
        "\n",
        "    def __init__(self, units, **kwargs): # units = units of the weight matrixs in each dense layer = units of the output\n",
        "        super().__init__(**kwargs)\n",
        "\n",
        "        self.units = units\n",
        "\n",
        "        self.forgetgate = tf.keras.layers.Dense(units, \n",
        "                                                kernel_initializer=tf.keras.initializers.Orthogonal(gain=1.0, seed=None), \n",
        "                                                activation='sigmoid')\n",
        "        self.inputgate1 = tf.keras.layers.Dense(units, \n",
        "                                                kernel_initializer=tf.keras.initializers.Orthogonal(gain=1.0, seed=None), \n",
        "                                                activation='sigmoid')\n",
        "        self.inputgate2 = tf.keras.layers.Dense(units, \n",
        "                                                kernel_initializer=tf.keras.initializers.Orthogonal(gain=1.0, seed=None), \n",
        "                                                activation='tanh')\n",
        "        self.outputgate = tf.keras.layers.Dense(units, \n",
        "                                                kernel_initializer=tf.keras.initializers.Orthogonal(gain=1.0, seed=None), \n",
        "                                                activation='sigmoid')\n",
        "    \n",
        "    @property\n",
        "    def state_size(self):\n",
        "        return [tf.TensorShape([self.units]), \n",
        "                tf.TensorShape([self.units])]\n",
        "    @property\n",
        "    def output_size(self):\n",
        "        return [tf.TensorShape([self.units])]\n",
        "    \n",
        "    def get_initial_state(self, inputs=None, batch_size=None, dtype=None):\n",
        "        return [tf.zeros([batch_size, self.units]), \n",
        "                tf.zeros([batch_size, self.units])]\n",
        "\n",
        "\n",
        "    # The LSTM-cell layer’s call method should take one (batch of) feature vector(s) as its input, \n",
        "    # along with the ”states”, a list containing the different state tensors of the LSTM cell (cell state and hidden state!).\n",
        "    def call(self, inputs, states):\n",
        "        # unpack the states\n",
        "        cell_s = states[0]\n",
        "        hidden_s = states[1]\n",
        "\n",
        "        concat_value = tf.concat([inputs, hidden_s], axis=-1) # Leon: or use a tuple\n",
        "\n",
        "        x1 = self.forgetgate(concat_value)\n",
        "        x1 = tf.math.multiply(x1, cell_s) # or use *\n",
        "\n",
        "        x2 = self.inputgate1(concat_value)\n",
        "        x3 = self.inputgate2(concat_value)\n",
        "\n",
        "        x3 = tf.math.multiply(x2, x3)\n",
        "        new_cell_s = tf.math.add(x1, x3)\n",
        "\n",
        "        x4 = self.outputgate(concat_value)\n",
        "        new_hidden_s = tf.math.multiply(x4, tf.math.tanh(new_cell_s))\n",
        "\n",
        "        return new_hidden_s, (new_hidden_s, new_cell_s)\n",
        "        # The returns should be the output of the LSTM, to be used to compute the model\n",
        "        # output for this time-step (usually the hidden state), as well as a list containing\n",
        "        # the new states (e.g. [new hidden state, new cell state])\n",
        "\n",
        "    def get_config(self):\n",
        "        return {\"hidden_units\": self.units}"
      ],
      "metadata": {
        "id": "GEWeGXBIvQA_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 20\n",
        "lr = 1e-2\n",
        "\n",
        "mymodel = BasicCNN_LSTM()\n",
        "\n",
        "loss = tf.keras.losses.MeanSquaredError()\n",
        "opti = tf.keras.optimizers.Adam(learning_rate=lr)\n",
        "\n",
        "mymodel.compile(loss=loss, \n",
        "                optimizer=opti, \n",
        "                metrics=['MAE']) # for accuracy - instead of tf.keras.metrics.MeanAbsoluteError()\n",
        "                      \n",
        "# save logs with Tensorboard\n",
        "EXPERIMENT_NAME = \"CNN_LSTM\"\n",
        "current_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
        "logging_callback = tf.keras.callbacks.TensorBoard(log_dir=f\".myModel/{EXPERIMENT_NAME}/{current_time}\")\n",
        "\n",
        "history = mymodel.fit(train_ds, \n",
        "                    validation_data=val_ds, \n",
        "                    epochs=epochs,\n",
        "                    batch_size=batch_size,\n",
        "                    callbacks=[logging_callback])\n",
        "\n",
        "# mymodel.load_weights(checkpoint_filepath)\n",
        "\n",
        "plt.plot(history.history[\"loss\"])\n",
        "plt.plot(history.history[\"val_loss\"])\n",
        "plt.plot(history.history[\"MAE\"])\n",
        "plt.plot(history.history[\"val_MAE\"])\n",
        "plt.legend(labels=[\"train_loss\",\"val_loss\", \"train_error(acc)\", \"val_error(acc)\"])\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"MSE(loss), MAE(acc)\")\n",
        "plt.savefig(f\"testing: e={epochs},lr={lr}.png\")\n",
        "plt.show()\n",
        "\n",
        "# save configs (e.g. hyperparameters) of your settings\n",
        "# checkpoint your model’s weights (or even the complete model\n",
        "mymodel.save('/content/MyModel')\n"
      ],
      "metadata": {
        "id": "WOftvqoL5fJn",
        "outputId": "d3e847dd-1d11-457a-a786-6d4f8e876381",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "157/157 [==============================] - 20s 99ms/step - loss: 30.5991 - MAE: 4.3451 - val_loss: 19.3776 - val_MAE: 3.4625\n",
            "Epoch 2/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 12.3192 - MAE: 2.6942 - val_loss: 9.5631 - val_MAE: 2.3615\n",
            "Epoch 3/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 8.5475 - MAE: 2.2196 - val_loss: 7.5453 - val_MAE: 2.0780\n",
            "Epoch 4/20\n",
            "157/157 [==============================] - 15s 98ms/step - loss: 7.3435 - MAE: 2.0519 - val_loss: 7.2913 - val_MAE: 2.0411\n",
            "Epoch 5/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 6.8828 - MAE: 1.9843 - val_loss: 6.3673 - val_MAE: 1.9010\n",
            "Epoch 6/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 6.1611 - MAE: 1.8661 - val_loss: 5.7859 - val_MAE: 1.8014\n",
            "Epoch 7/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 5.8480 - MAE: 1.8087 - val_loss: 5.4592 - val_MAE: 1.7519\n",
            "Epoch 8/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 5.4088 - MAE: 1.7500 - val_loss: 5.1848 - val_MAE: 1.7131\n",
            "Epoch 9/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 5.0281 - MAE: 1.6773 - val_loss: 4.7899 - val_MAE: 1.6400\n",
            "Epoch 10/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 4.6286 - MAE: 1.6012 - val_loss: 4.2300 - val_MAE: 1.5388\n",
            "Epoch 11/20\n",
            "157/157 [==============================] - 14s 86ms/step - loss: 4.4020 - MAE: 1.5609 - val_loss: 4.0894 - val_MAE: 1.5090\n",
            "Epoch 12/20\n",
            "157/157 [==============================] - 14s 86ms/step - loss: 4.2694 - MAE: 1.5355 - val_loss: 3.9558 - val_MAE: 1.4747\n",
            "Epoch 13/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 3.9802 - MAE: 1.4782 - val_loss: 3.7297 - val_MAE: 1.4185\n",
            "Epoch 14/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 3.9453 - MAE: 1.4684 - val_loss: 3.6002 - val_MAE: 1.3987\n",
            "Epoch 15/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 3.7861 - MAE: 1.4391 - val_loss: 3.6008 - val_MAE: 1.3869\n",
            "Epoch 16/20\n",
            "157/157 [==============================] - 12s 74ms/step - loss: 3.6242 - MAE: 1.4067 - val_loss: 3.4271 - val_MAE: 1.3631\n",
            "Epoch 17/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 3.5024 - MAE: 1.3701 - val_loss: 3.3459 - val_MAE: 1.3533\n",
            "Epoch 18/20\n",
            "157/157 [==============================] - 14s 87ms/step - loss: 3.2929 - MAE: 1.3404 - val_loss: 3.3931 - val_MAE: 1.3690\n",
            "Epoch 19/20\n",
            "157/157 [==============================] - 12s 75ms/step - loss: 3.2614 - MAE: 1.3284 - val_loss: 3.0944 - val_MAE: 1.3042\n",
            "Epoch 20/20\n",
            "157/157 [==============================] - 12s 75ms/step - loss: 3.2146 - MAE: 1.3256 - val_loss: 2.9335 - val_MAE: 1.2634\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU1f34/9d79kz2hLBnQZQdAUFF0LqvtaJ+ROq+tdSW1uXjx29xaavWtrZaW7dqtfKp8LN1wVL9tLauWKq4sAiySsACCVtC9pBMJjNzfn/cmZB9I5NJMu+n3sddZ+47k/A+d8499xwxxqCUUip+2GIdgFJKqd6liV8ppeKMJn6llIozmviVUirOaOJXSqk444h1AJ0xaNAgk5eXF+swlFKqX1mzZs1BY0xW8+39IvHn5eWxevXqWIehlFL9iojsam27VvUopVSc0cSvlFJxRhO/UkrFmX5Rx6+Uip76+noKCwvx+XyxDkV1k8fjYeTIkTidzk4dr4lfqThXWFhIcnIyeXl5iEisw1FdZIyhpKSEwsJCRo0a1anXaFWPUnHO5/ORmZmpSb+fEhEyMzO79I1NE79SSpN+P9fV39+ATvzvbz3A7z7YHuswlFKqTxnQif/D/BKeeG87OuaAUkodFrXELyIeEflMRNaLyCYRuT+8fZSIfCoi20XkZRFxRSuG7IwEauuDHKz2R+sUSqkeUF5ezu9+97suv+6CCy6gvLy8y6+7/vrrWbp0aZdfN1BE84q/DjjDGDMFmAqcJyIzgV8CvzHGHA2UATdFK4CcDC8ABWU10TqFUqoHtJX4A4FAu6978803SUtLi1ZYA1bUmnMaq36lOrzqDE8GOAO4Mrz9BeA+4OloxJAdSfylNRyXkx6NUyg1oNz/f5vYvLeyR99zwvAUfvKNie0es3DhQnbs2MHUqVNxOp14PB7S09PZunUr27Zt4+KLL6agoACfz8ett97K/PnzgcP9eFVXV3P++edz8skns3LlSkaMGMHrr79OQkJCh/G99957/M///A+BQIDjjz+ep59+GrfbzcKFC3njjTdwOBycc845PPLII7z66qvcf//92O12UlNTWbFiRY98Rr0tqu34RcQOrAGOBp4CdgDlxphIMV4IjGjjtfOB+QA5OTndOv/IdOuXXlCqV/xK9WUPPfQQGzduZN26dXzwwQd8/etfZ+PGjQ3t0hctWkRGRga1tbUcf/zx/Nd//ReZmZlN3iM/P58///nPPPfcc1x++eW89tprXH311e2e1+fzcf311/Pee+8xZswYrr32Wp5++mmuueYali1bxtatWxGRhuqkBx54gLfeeosRI0Z0q4qpr4hq4jfGBIGpIpIGLAPGdeG1zwLPAsyYMaNbd2e9LgeDktwUlNZ25+VKxZ2Orsx7ywknnNDkYaTHH3+cZcuWAVBQUEB+fn6LxD9q1CimTp0KwPTp09m5c2eH5/nyyy8ZNWoUY8aMAeC6667jqaee4vvf/z4ej4ebbrqJCy+8kAsvvBCA2bNnc/3113P55Zdz6aWX9sSPGhO90qrHGFMOLAdOAtJEJFLgjAT2RPPc2RkJWsevVD+TmJjYsPzBBx/w7rvv8vHHH7N+/XqmTZvW6sNKbre7Ydlut3d4f6A9DoeDzz77jMsuu4y//e1vnHfeeQA888wzPPjggxQUFDB9+nRKSkq6fY5YimarnqzwlT4ikgCcDWzBKgAuCx92HfB6tGIAyE73auJXqo9LTk6mqqqq1X0VFRWkp6fj9XrZunUrn3zySY+dd+zYsezcuZPt263nfZYsWcKpp55KdXU1FRUVXHDBBfzmN79h/fr1AOzYsYMTTzyRBx54gKysLAoKCnoslt4UzaqeYcAL4Xp+G/CKMeZvIrIZeElEHgQ+B56PYgzkZHj5+4Z9BIIhHPYB/diCUv1WZmYms2fPZtKkSSQkJDBkyJCGfeeddx7PPPMM48ePZ+zYscycObPHzuvxePjf//1f5s6d23Bz9+abb6a0tJQ5c+bg8/kwxvDoo48CcOedd5Kfn48xhjPPPJMpU6b0WCy9SfrDw00zZsww3R2B6+VVu/nhaxv49/87vaGVj1LqsC1btjB+/PhYh6GOUGu/RxFZY4yZ0fzYAX8JnJ1+uEmnUkqpOOiWOXKVv7u0hlkxjkUp1bsWLFjARx991GTbrbfeyg033BCjiPqGAZ/4h6V6sNtEb/AqFYeeeuqpWIfQJw34qh6H3cbwNI+25VdKqbABn/hBm3QqpVRjcZH4czK8enNXKaXC4iLxZ2d4OVjtp8bf/Sf5lFJqoIiLxB/prK2wTOv5lRoIkpKS2ty3c+dOJk2a1IvR9D9xkfgj/fLvLtHqHqWUGvDNOaFRv/x6g1ep9v1jIezf0LPvOXQynP9Qu4csXLiQ7OxsFixYAMB9992Hw+Fg+fLllJWVUV9fz4MPPsicOXO6dGqfz8d3v/tdVq9ejcPh4NFHH+X0009n06ZN3HDDDfj9fkKhEK+99hrDhw/n8ssvp7CwkGAwyI9+9CPmzZvX7R+7L4uLxJ+Z6CLBadcmnUr1UfPmzeO2225rSPyvvPIKb731FrfccgspKSkcPHiQmTNnctFFFyEinX7fp556ChFhw4YNbN26lXPOOYdt27bxzDPPcOutt3LVVVfh9/sJBoO8+eabDB8+nL///e+A1TncQBUXiV9EyM5IYLe27FGqfR1cmUfLtGnTKCoqYu/evRQXF5Oens7QoUO5/fbbWbFiBTabjT179nDgwAGGDh3a6ff98MMP+cEPfgDAuHHjyM3NZdu2bZx00kn87Gc/o7CwkEsvvZRjjjmGyZMnc8cdd/DDH/6QCy+8kFNOOSVaP27MxUUdP1j1/IVa1aNUnzV37lyWLl3Kyy+/zLx583jxxRcpLi5mzZo1rFu3jiFDhrTaD393XHnllbzxxhskJCRwwQUX8P777zNmzBjWrl3L5MmTuffee3nggQd65Fx9Udwk/pHpVlv+/tAbqVLxaN68ebz00kssXbqUuXPnUlFRweDBg3E6nSxfvpxdu3Z1+T1POeUUXnzxRQC2bdvG7t27GTt2LF999RVHHXUUt9xyC3PmzOGLL75g7969eL1err76au68807Wrl3b0z9inxEXVT1g3eA95A9SVlNPRqIr1uEopZqZOHEiVVVVjBgxgmHDhnHVVVfxjW98g8mTJzNjxgzGjev0yK0Nvve97/Hd736XyZMn43A4+OMf/4jb7eaVV15hyZIlOJ1Ohg4dyt13382qVau48847sdlsOJ1Onn766Sj8lH3DgO+PP+KdzQf49uLV/HXBbKZmp/VQZEr1f9of/8Cg/fG3IjvDeohLu25QSsW7+KnqSde2/EoNJBs2bOCaa65pss3tdvPpp5/GKKL+I24Sf6LbQUaiS6/4lRogJk+ezLp162IdRr8UN1U9YN3g1Ye4lFLxLr4Sf3qCVvUopeJefCX+DC97ymoJhvp+SyallIqWuEr8ORleAiHDvgqt7lFKxa+4SvwNLXu0nl+pPqW8vJzf/e53XX7dBRdcQHl5eRQiOnK//e1vWbx4cY+935NPPsmiRYt65L3iK/FH2vJrPb9SfUpbiT8QaH/UvDfffJO0tOg8kNn83B3F0vi4QCDAokWLuPLKK3ssnhtvvJEnnniiR94ras05RSQbWAwMAQzwrDHmMRG5D/g2UBw+9G5jzJvRiqOx4WkJ2EQf4lKqLb/87JdsLd3ao+85LmMcPzzhh+0es3DhQnbs2MHUqVNxOp14PB7S09PZunUr27Zt4+KLL6agoACfz8ett97K/PnzAcjLy2P16tVUV1dz/vnnc/LJJ7Ny5UpGjBjB66+/TkJCQqvn27FjBwsWLKC4uBiv18tzzz3HuHHjuP766/F4PHz++efMnj2b0tLSJuvXXnstN998MzU1NYwePZpFixaRnp7OaaedxtSpU/nwww+54oormDx5MscddxwOh5Vin3vuOZ599ln8fj9HH300S5Yswev1cuDAAW6++Wa++uorAJ5++mlmzZrF4sWLeeSRRxARjj322Ibj8/Ly+OyzzzjhhBOO6HcSzSv+AHCHMWYCMBNYICITwvt+Y4yZGp56JekDOO02hqUmaOJXqo956KGHGD16NOvWrePhhx9m7dq1PPbYY2zbtg2ARYsWsWbNGlavXs3jjz9OSUlJi/fIz89nwYIFbNq0ibS0NF577bU2zzd//nyeeOIJ1qxZwyOPPML3vve9hn2FhYWsXLmSRx99tMX6tddeyy9/+Uu++OILJk+ezP3339/wOr/fz+rVq7njjjv46KOPmD59esO+Sy+9lFWrVrF+/XrGjx/P888/D8Att9zCqaeeyvr161m7di0TJ05k06ZNPPjgg7z//vusX7+exx57rOF9ZsyYwb///e9ufsqHRe2K3xizD9gXXq4SkS3AiGidr7OyMxIo0LF3lWpVR1fmveWEE05g1KhRDeuPP/44y5YtA6CgoID8/HwyMzObvGbUqFFMnToVgOnTp7Nz585W37u6upqVK1cyd+7chm11dXUNy3PnzsVut7dYr6iooLy8nFNPPRWA6667rsl7NB6ta9++fU36zdm4cSP33nsv5eXlVFdXc+655wLw/vvvN9wHsNvtpKamsnjxYubOncugQYMAyMjIaHifwYMHs3XrkX8j65Und0UkD5gGfArMBr4vItcCq7G+FZS18pr5wHyAnJycHoslO93Lv7YVd3ygUipmEhMTG5Y/+OAD3n33XT7++GO8Xi+nnXZaq/3yu93uhmW73U5tbesXeKFQiLS0tDaf+m187tbWOxNzQkJCkxivv/56/vrXvzJlyhT++Mc/8sEHH3TqPZvz+XxtVl91RdRv7opIEvAacJsxphJ4GhgNTMX6RvDr1l5njHnWGDPDGDMjKyurx+LJyfBSVFWHrz7YY++plDoyycnJVFVVtbqvoqKC9PR0vF4vW7du5ZNPPjmic6WkpDBq1CheffVVAIwxrF+/vsPXpaamkp6e3lDVsmTJkoar/+bGjx/P9u3bG9arqqoYNmwY9fX1DeMDAJx55pkN3T8Hg0EqKio444wzePXVVxuqs0pLSxuO37ZtG5MmTeriT9xSVBO/iDixkv6Lxpi/ABhjDhhjgsaYEPAccGR3KbooMvC6jsalVN+RmZnJ7NmzmTRpEnfeeWeTfeeddx6BQIDx48ezcOFCZs6cecTne/HFF3n++eeZMmUKEydO5PXXX+/U61544QXuvPNOjj32WNatW8ePf/zjVo87//zzWbFiRcP6T3/6U0488URmz57dZFyBxx57jOXLlzN58mSmT5/O5s2bmThxIvfccw+nnnoqU6ZM4b//+78bjv/oo484++yzu/lTN2KMicoECFarnt822z6s0fLtwEsdvdf06dNNT1m9s8Tk/vBv5v0tB3rsPZXqzzZv3hzrEAakiy++2Gzbtq3H3m/t2rXm6quvbnN/a79HYLVpJadGs45/NnANsEFEIpVpdwNXiMhUrCaeO4HvRDGGFiJX/DrwulIqmh566CH27dvHMccc0yPvd/DgQX7605/2yHtFs1XPh1hX/c31WvPN1mQlufE4bdqkU6k4sGDBAj766KMm22699VZuuOGGqJ977NixjB07tsfer0eqeMLipj/+CBGxBl7XOn6lBrynnnoq1iH0SXHVZUNEjvbLr5SKY3GZ+LPTrad3TT8YaF4ppXpafCb+DC9VdQEqautjHYpSSvW6uEz8I7V7ZqVUHIvLxJ+jTTqV6teSkpJiev7a2lpOPfVUgsGe6QHA7/fzta99rdNdPx+pTiV+EbGJyDQR+bqInCEig6MdWDRpv/xKxTdjDKFQqM31tkQS86JFi7j00kubdOZ2JFwuF2eeeSYvv/xyj7xfR9ptzikio4EfAmcB+Vh96HuAMSJSA/weeMFY3S/0G8keJ2lep7blV6qZ/T//OXVberY/fvf4cQy9++52j1m4cCHZ2dksWLAAgPvuuw+Hw8Hy5cspKyujvr6eBx98kDlz5nTqnA8//DCvvPIKdXV1XHLJJdx///3s3LmTc889lxNPPJE1a9bwu9/9jvnz5zesv/nmmzz55JP84x//QES49957mTdvHh988AE/+tGPmowP8OKLL/KnP/0JsHr7nDNnTqtxttavflt98F988cXcddddXHXVVd39qDuto3b8D2J1qvYd06wJTPiq/0qsp3NfiE540ZOT4dWqHqX6iHnz5nHbbbc1JP5XXnmFt956i1tuuYWUlBQOHjzIzJkzueiiixBp7bnQw95++23y8/P57LPPMMZw0UUXsWLFCnJycsjPz+eFF15g5syZ7Ny5s8n6a6+9xrp161i/fj0HDx7k+OOP52tf+xoAa9euZePGjYwaNQq/389XX31FXl4eAB6Ph2XLlrWIc/PmzTz44IOsXLmSQYMGNXS2FumDf9myZQSDQaqrqwGYNGkSq1atitIn3FS7id8Yc0U7+4qA3/Z4RL0kO93L5n2VsQ5DqT6loyvzaJk2bRpFRUXs3buX4uJi0tPTGTp0KLfffjsrVqzAZrOxZ88eDhw4wNChQ9t9r7fffpu3336badOmAdYVeX5+Pjk5OeTm5jbp5K3xemT0LLvdzpAhQzj11FNZtWoVKSkpTcYHOHjwYJPhHo0x3H333S3ifP/991vtV7+1Pvgjyy6Xi6qqKpKTk3viY21Tp57cFZEFWD1slofX04ErjDFdHx25jxiZkcA7mw8QChlstvavIJRS0Td37lyWLl3K/v37mTdvHi+++CLFxcWsWbMGp9NJXl5eq/3wN2eM4a677uI732naDdjOnTuj0td+d+NsTV1dHR6Pp1uv7YrOtur5diTpAxhr4JRvRyek3pGd7sUfDHGgqnu/IKVUz5o3bx4vvfQSS5cuZe7cuVRUVDB48GCcTifLly9n165dnXqfc889l0WLFjVUoezZs4eioqIOX3fKKafw8ssvEwwGKS4uZsWKFa2ObZuenk4wGGxI7m3F2Va/+q31wQ9QUlLCoEGDcDqdnfo5j0RnE79dGlWsiYgdcEUnpN7R0KSzROv5leoLJk6cSFVVFSNGjGDYsGFcddVVrF69msmTJ7N48eIm/di355xzzuHKK6/kpJNOYvLkyVx22WVtDvLS2CWXXMKxxx7LlClTOOOMM/jVr37VZrXSOeecw4cffgjQZpxt9avfWh/8AMuXL+frX/96p37GIyWd6bZARB4GcrFa8YDVlXKBMeaOKMbWYMaMGWb16tU9+p7/OXiI0x/5gEfmTuGy6SN79L2V6k+2bNnSZHxY1bG1a9fym9/8hiVLlvTYe1566aU89NBDjBkzpluvb+33KCJrjDEzmh/b2d45f4g1/u13w+vvAH/oVnR9xPA0DyJok06lVJcdd9xxnH766QSDwR5py+/3+7n44ou7nfS7qrOJPwF4zhjzDDRU9biBfps13Q47w1I8mviV6qc2bNjANddc02Sb2+3m008/7ZXz33jjjT32Xi6Xi2uvvbbH3q8jnU3872E9xFUdXk8A3gZmRSOo3jIyQ/vlVwqsljAdtY/vayZPnsy6des6PjAOdLWn4c7e3PUYYyJJn/Cyt0tn6oOy07VffqU8Hg8lJSXaTXk/ZYyhpKSkS81AO3vFf0hEjjPGrAUQkelAv8+YORle/lLlw1cfxOPsmT43lOpvRo4cSWFhIcXFxbEORXWTx+Nh5MjON1LpbOK/DXhVRPZijaM7FJjX9fD6luyMBIyBPeW1jM6KbW9/SsWK0+lseCpVxYdOJX5jzCoRGQdERg7+0hjT70cxyc6I9Mtfo4lfKRU3ujLY+lhgAlbvnMeJCMaYxdEJq3dkRwZkKev3tVZKKdVpne2r5yfAaViJ/03gfOBDoF8n/sHJblwOmzbpVErFlc626rkMOBPYb4y5AZgCpEYtql5iswkjwwOvK6VUvOhs4q8ND7YSEJEUoAjIjl5YvSc7XdvyK6XiS2cT/2oRSQOeA9YAa4GP23uBiGSLyHIR2Swim0Tk1vD2DBF5R0Tyw/P0I/oJjlBOhrblV0rFl04lfmPM94wx5eEuG84GrgtX+bQnANxhjJkAzAQWiMgEYCHwnjHmGKwnghd2P/wjl52RQEVtPRW1/b6RklJKdUq7iV9E8ppvM8bsNMZ8Ed4vItLqUwPGmH2RB76MMVXAFmAEMIfDQzW+AFzc3eB7QkPLHq3nV0rFiY6u+B8WkddE5FoRmSgig0UkR0TOEJGfAh8BHfbnGi5ApgGfAkOMMfvCu/YDQ9p4zXwRWS0iq6P5RGGkLX+h1vMrpeJER2Puzg1Xz1wF3AgMw+qRcwtWs86fGWPaHcJKRJKA14DbjDGVjTuCMsYYEWm1gxBjzLPAs2D1x9/pn6iLIolfB15XSsWLDtvxG2M2A/d0581FxImV9F80xvwlvPmAiAwzxuwTkWFYLYSiJxQCW9tfbFITnKR4HHqDVykVNzqq4/9/jZbnNtv38w5eK8DzwBZjzKONdr0BXBdevg54vSsBd8m798PvT+nwsGztnlkpFUc6quP/ZqPlu5rtO6+D184GrgHOEJF14ekC4CHgbBHJx+rj/6GuBNwl3gw4sBGqDrR7mNWkUxO/Uio+dFTVI20st7behDHmw3aOObOD8/aM3PA4MbtXwsRL2jwsO8PLe1uLCIUMNlv/GoxCKaW6qqMrftPGcmvrfc/QKeBMhF0r2z0sOz0BfyBEcXVdLwWmlFKx09EV/xQRqcS6ck8ILxNe7/xwL7Fid0DOibDzo3YPa9w985CUvv9jKaXUkWj3it8YYzfGpBhjko0xjvByZN3ZW0EekdxZULQJakrbPESbdCql4kln++ppICKJInK1iPw9GgH1uNzZ1nz3J20eMiItAUCbdCql4kKnEr+IuETkEhF5FdiHdXP2mahG1lOGHwd2N+xqu7rH47QzJMWtTTqVUnGh3Tp+ETkHuAI4B1iONfDK8Z3ooK3vcHpg5IwOb/DmZHi1qkcpFRc6uuL/J3AUcLIx5mpjzP8BoeiH1cNyZ8G+9VBX1eYh2eleCjXxK6XiQEeJ/zisfvffDfedfxNgj35YPSx3FpggFHzW5iEjM7zsq/ThD/S/ck0ppbqio1Y964wxC40xo4GfAFMBp4j8Q0Tm90qEPWHkCSD2dqt7cjK8GAN7y/UGr1JqYOt0qx5jzEpjzA+AkcBvsAZX6R/cSTB8aruJPzvdatmj9fxKqYGuo5u7x7Wx6yDwZM+HE0W5s+DT30O9z7rh20zDQ1zaskcpNcB19OTuamAjVqKHpn3vGOCMaAQVFbmzYeUTsGcN5M1usXtIigeX3aZt+ZVSA15Hif+/gcuAWuAlYJkxpjrqUUVDzkxArOqeVhK/3SaMSE/QXjqVUgNeRzd3f2uMORn4AZANvCcir4jI1F6JriclpMOQie0+yDUyPUGrepRSA16nbu4aY77CGjDlbeAEYEw0g4qa3FlWk85gfau7s7VffqVUHOhoBK6jRORuEfkUuB9YD4w3xrzSK9H1tNxZUH8I9n3R6u6cDC9lNfVU+VovGJRSaiDoqI5/O/AF1tV+JZADfDcyYHqzIRX7vpzwwCy7PoKR01vszk6PdM9cy4Th/aPzUaWU6qqOqnoeAJZhddOQBCQ3m/qX5CGQeXSb7fmzM8K9dGo9v1JqAGv3it8Yc18vxdF7cmfB5tchFAJb03Ivp9GALEopNVB1uT/+fi93NvgqoGhzi12pCU6S3Q5N/EqpAS0OE3+knr9ldY+IMDLDS0GZPsSllBq44i/xp+VAanab7flzMvQhLqXUwNatxC8ic0TkxJ4OptfkzrKu+I1psSs73UtBWQ2mlX1KKTUQdPeK/0TgXhH5R08G02tyZ8GhIijZ0WJXdoYXX32I4uq6GASmlFLR163Eb4y52xjzDWPM+W0dIyKLRKRIRDY22nafiOwRkXXh6YLunP+IRQZgb6W6p6FJp3bWppQaoDo72Prg8GDrC0TkRhE5QUQ6eu0fgfNa2f4bY8zU8PRmVwPuEZlHQ2JWqzd4I006C7Utv1JqgOqoP/7TgYVABvA5UAR4gIuB0SKyFPi1Maay+WuNMStEJK+nA+4RIofr+ZsZGX56d3eJJn6l1MDUUZcNFwDfNsbsbr5DRBzAhcDZwGtdOOf3ReRarL7+7zDGlLV2UHhox/kAOTk5XXj7TsqdbT3IVb7baukT5nHayUp269O7SqkBq6Nume9sLemH9wWMMX81xnQl6T8NjMYau3cf8Ot2zv2sMWaGMWZGVlZWF07RSQ3t+T9usSsnw6t1/EqpAauzdfy3ikiKWJ4XkbUick5XT2aMOWCMCRpjQsBzWF08x8bgCeBJbf0Gb3qCjr2rlBqwOtuq58ZwPf45QDpwDfBQV08mIsMarV6CNaxjbNjskHNSq/X82Rle9lXUUh8MxSAwpZSKrs4m/shYuxcAS4wxm2g6/m7LF4j8GfgYGCsihSJyE/ArEdkgIl8ApwO3dzPunpE7C0ryobqoyebsdC8hA/vKfTEKTCmloqejm7sRa0TkbWAUcJeIJGN11dwmY8wVrWx+vovxRVdDe/6VMPHihs3ZkV46y2rIyfTGIjKllIqazl7x34TVrPN4Y0wN4ARuiFpUvWXYFHB6W1T3RB7i0np+pdRA1NnEfxLwpTGmXESuBu4FKqIXVi+xOyH7hBaJf1hqAg6baGdtSqkBqbOJ/2mgRkSmAHcAO4DFUYuqN+XOhgMbofbw4wR2mzAiPUG7Z1ZKDUidTfwBY3VXOQd40hjzFP1x6MXW5M4CDOz+tMnm7HSvVvUopQakzib+KhG5C6sZ59/D/fQMjNHIR0wHu6tFe/7sjAQKNfErpQagzib+eUAdVnv+/cBI4OGoRdWbnAlW8m9xg9dLySE/h+oCMQpMKaWio1OJP5zsXwRSReRCwGeMGRh1/GBV9+xbB3XVDZuy0yO9dGo9v1JqYOlslw2XA58Bc4HLgU9F5LJoBtarcmdBKACFqxo2Rdryaz2/Umqg6ewDXPdgteEvAhCRLOBdYGm0AutV2SeC2KzqntGnW5vSIwOyaOJXSg0sna3jt0WSflhJF17b97mTrYe5GtXzZyS6SHTZtXtmpdSA09kr/t1n0tsAACAASURBVH+KyFvAn8Pr84DYjJ4VLbmz4bPnIFAHDjciQnaGV6/4lVIDTmdv7t4JPAscG56eNcb8MJqB9brcWRCsgz1rGzaNTNd++ZVSA09nr/gJD7jSlUFX+peck6z5ro8g11o+ZkgSH3xZxPaiao4enBTD4JRSque0e8UvIlUiUtnKVCUiLcbZ7de8GdbgLI3q+W+cPQqvy849yzZgPbislFL9X0dDLyYbY1JamZKNMSm9FWSvyZ0FBZ9C0HpoKyvZzd0XjOfT/5Ty6urCGAenlFI9o6Mr/g7rNzpzTL+ROwv81bD/i4ZNl8/I5oS8DH725hYOVtfFMDillOoZHd3cfV1Efi0iXxORxMhGETlKRG4Kt/Q5L7oh9qKcyADsh6t7bDbh55dOotYf5Kd/2xyjwJRSqud0VNVzJvAe8B1gk4hUiEgJ8P8BQ4HrjDED4yEugJRhkHFUi357jh6czHdPG83r6/bywZdFbbxYKaX6hw6bcxpj3jTGXGWMyTPGpBpjMo0xs4wxPwv34TOw5M6C3Ssh1HRkye+dPpqjshL50esbqfUHYxScUkoduY7q+K9utDy72b7vRyuomMqdbQ3KUry1yWa3w87PL5lMQWktv31vW4yCU0qpI9fRFf9/N1p+otm+G3s4lr4hN1LP/1GLXTOPymTejGz+8O//sGlv/x95UikVnzpK/NLGcmvrA0NaLqSMaFHPH3HXBeNI9zq5+y8bCIa0bb9Sqv/pKPGbNpZbWx8YRKyr/l0fQSsPbaV5XfzowgmsL6xgycc7ez08pZQ6Uh0l/nEi8oWIbGi0HFkf2wvxxUbuLKg+AKVftbr7oinD+dqYLB5+60v2lmtfPkqp/qWjxD8e+AZwYaPlyPqE6IYWQ7nh+9it1PMDiAg/u3gSQWP4yRubejEwpZQ6ch2149/VeAKqgeOAQeH1NonIIhEpEpGNjbZliMg7IpIfnqf3yE/R0waNAW9mm/X8YI3QdftZY3hn8wH+uXHgtWpVSg1cHTXn/JuITAovDwM2YrXmWSIit3Xw3n+k5VO9C4H3jDHHYD0YtrA7QUdd43r+dtx48ijGD0vhJ29spNJX30vBKaXUkemoqmeUMSZyxX4D8I4x5hvAiXTQnNMYswIobbZ5DvBCePkF4OKuhduLcmdD+W4oL2jzEKfdxi8unUxRVR2PvPVlLwanlFLd11Hib3wZeybhUbeMMVVAqNVXtG+IMWZfeHk/MKStA0VkvoisFpHVxcXF3TjVEYq059/9cbuHTc1O47qT8ljyyS7W7CrrhcCUUurIdJT4C0TkByJyCVbd/j8BRCQBcB7JiY3VwX2bTUKNMc8aY2YYY2ZkZWUdyam6Z8gkcKd0WN0D8D/njmVoioe7/7KB+mB3ykOllOo9HSX+m4CJwPXAPGNMeXj7TOB/u3G+A+F7BZF7Bn23xzObHXJmwo73wdf+mDNJbgf3XzSRLw9U8dy/W28CqpRSfUVHrXqKjDE3G2PmGGPebrR9uTHmkW6c7w3guvDydcDr3XiP3jPjRqjcC4vOter723HOxKGcN3Eoj72bz66SQ70UoFJKdZ20N6SgiLzR3ouNMRe189o/A6cBg4ADwE+AvwKvADnALuByY0zzG8AtzJgxw6xevbqjw6Ljqw/g5WvB4YJv/hmyj2/z0P0VPs569F9MzU5jyU0nIDIwe7VQSvUPIrLGGDOjxfYOEn8xUAD8GfiUZv3zGGP+1cNxtiqmiR+geBv8aS5U7oNLnoZJ/9XmoYs/3smPX9/Eb+ZN4ZJpI3svRqWUaqatxN9RHf9Q4G5gEvAYcDZw0Bjzr95K+n1C1hj41vsw4jhYeiP86+FW+/EBuOrEXKblpPHTv22h9JC/lwNVSqmOdVTHHzTG/NMYcx3WDd3twAcDti/+9iRmwrWvw7HzYPmDsOxmCLQcg9duE35x6WQqa+v5+ZtbYhCoUkq1r8MRuETELSKXYg23uAB4HFgW7cD6JIcbLvk9nH4vfPESLJ4Dh0paHDZuaArf/tpRLF1TyMrtB2MQqFJKta2jLhsWAx9jteG/3xhzvDHmp8aYPb0SXV8kAqfeCZctgj1r4Q9nWvcAmrn1zGPIzfRy+yvr+Ovne7TvfqVUn9HRzd0QEGmb2PhAwXoGKyWKsTWI+c3dthSsgpeugKAfLl8MR53WZPeGwgruXLqerfurGJ2VyG1njeHrk4dhs2lrH6VU9HWrVU9f0WcTP0DZLvjTPCjJh6//GqZf32R3KGT4x8b9/PbdbeQXVTNmSBK3nTWG8yYO1QJAKRVVmvijyVcJS2+A7e/CrB/AWfdbT/42EgwZ/r5hH799dxtfFR9i3NBkbjtrDOdOHKLt/ZVSUaGJP9qCAfjnQlj1HIz9OvzXc+BKbHlYyPB/6/fy2Hv5/OfgISYOT+G2s8Zw1vjBWgAopXqUJv7e8unvrQJg6GS44iVIGd7qYYFgiNfXWQXA7tIajh2Zyu1njeG0sVlaACileoQm/t607W2r6sedDJf9LwyfBk5Pq4fWB0MsW7uHx9/Pp7CslqnZadx+9hi+dswgLQCUUkdEE39v27/RuulbWWitJ6RDyghIHmZ9C0gZHl4eASnD8HuH8trmap5cvoM95bVMz03n9rPGMPvoTC0AlFLdook/Fg6VQP5bVg+flXuhah9U7rH6/DlUTIvhCBwJhJKHUUwGn1d42elPpTZ1NLWjL+DonOFMGp7KMUOScNo7fO5OKaU08fc5AT9U77cKgco94ULhcAERqtiDqdyH3dRTY9z8PXgiLwdP4wvbeMYOTWHSiBQmDE9l0vAUxg9LweO0d3xOpVRc0cTfHxkDe9YSWrsYNryKrf4QJZ5c3nKfzR8qT+SrWqvVkE3g6MFJTBqeyoThKUwaYc1TPEc0SJpSqp/TxN/f1VXD5r/C2iVQ8AnG5sA36mw2Db2Ef4cms2HfITbtreBA5eGO43IzvUwZmcYZ4wZz2tgs0ryuGP4ASqnepol/ICneBp8vhnV/hpqDkDwcpl0F066myDGUTXsr2by3ko17Kli1s5SD1X7sNuH4vHTOGj+Es8YPIW9Qy2cMlFIDiyb+gSjgh23/hM+XWE8NmxCMOhWOuxbGXQhOD6GQYV1hOe9tOcC7m4v48kAVYFUNnTV+CGdPGMzU7HTs2n2EUgOOJv6BrmIPrPuT9U2gfDd40qyxA6Z8E4ZNBZvVEqigtIZ3txzg3S0H+PSrUgIhQ0aiizPGDeas8UM45ZhBJLodMf5hlFI9QRN/vAiFYOcK617AljesnkM9qZBzEuTOtqZhU8DuoKK2nhXbinl3ywGWby2i0hfA5bAxa3RmQ5XQ0NTWHzxTSvV9mvjjUU0p5L8Duz6EXSuhZLu13ZUE2SdC7izIOxmGT6NenKzaWcp7W4p4d8sBdpXUADBmSBKjBiWSne4lJ9NLdoaX7HQvI9MTtAmpUn2cJn4FVfth10dWIbDzIygODw3p8MDI461vA3mzMSNmsL0syLtbili9s5TdpTUUlNXgqw81ebuhKR6yMxLIzvCSEy4QcjKt5awkt3Y7rVSMaeJXLR0qgd3hQmDXR7B/A2DA5oQR0yFvNow8ATKPxqRlU1xrKCitYXdpDbtLaikos5YLSmvYX+lrMv68y2EjOz2BvMxExg1LZsKwVMYPSyY3M1FvJCvVSzTxq47VlkPBp7DzQ6sg2LsOTNDaJzZIGQkZeZA+CjKOgoxR4eVR1Nm97CmrbSgICspq2V1Sw1cHq9lRfKhh6MkEp52xQ5OZEH7ieMKwZMYNTdEbykpFgSZ+1XV11XBgI5T+B8r+Y81Lv7KWa5oNMu8dZBUEGUc1FAakj4L0PHzuDLYX17B5byWb91WyJTxV+gKANYxxboaX8cMihUEK44enMDzVox3UKXUENPGrnuWrbFkYlP4HynZCRSFNOqCzu6xeSFNHNkwmZQQH7Vnk+9JYX5nE+qIgW/ZXNtxUBkhNcDJuaDJHD05qMg1N0QJBqc7oU4lfRHYCVUAQCLQWWGOa+PuZQJ01FnFZo4Kgco81r9gDVXuth80a86RCykgCySMoc2RRGMpkmy+V9VUprCjPpNB3uFlpktvB6KxERkcKgyxrnpPhxaE9lyrVoK3EH8uK1dONMQdjeH4VLQ43ZI2xptYEA1bPpBWFh6dwweCoKCSrYjVZtaVMA+ZFXpKRRVXK0ex35ZJvRrLON5QV+YP4y9rD/Q+57DbyBnkbCoPRg5M4ZnAyR2UlatNTpRrRO2qq99kdh6t92uKvsQqD0v/AwS+xF20lrXgrafv/zjh/Fd8IHxbKyOJQ6tHsd49iByNY7xvGyj2D+OdGB+H7ydgE8jITOWZIEmOGJHPMkOSG5xPcDi0QVPyJVVXPf4AyrIrg3xtjnm3lmPnAfICcnJzpu3bt6t0gVd9kjPUNofhL6zmE4q1QtNVa91cdPiwxi5rUozngGc1mx3g+rDuaz0o97CqpaWhhZLcJeZneJoXBmCHJ5GUm4nJolZHq//paHf8IY8weERkMvAP8wBizoq3jtY5fdcgY6xtC0VarMCjeYhUGBzZBffiGcVoOweyTKEqbxhbnBNbWDGFbUTX5RdXsKjnU8A3BYRNGDUpkzJBkRmclMjjFw+BkN1mNJv2moPqDPpX4mwQgch9QbYx5pK1jNPGrbgvWWw+m7f4Edn9sTYeKrX0JGZAzE3JmUjfiRHbYR7PtoJ9tB6rYdqCa/KIqdpfW0No/kdQEZ5PCYHDD3NNkPTXBqS2QVMz0mcQvIomAzRhTFV5+B3jAGPPPtl7T3cT/r4J/saZoDd+f+n1cdh2ERGF9Myj96nAhsOtjKN1h7XN4YMQMqzDIPQlGnkC9M4nSQ36KKusorvZZ86o6iqvrwtvqKKqyttcFQi1O57AJyR4HSR4HSW4nyR4HyW5rPbnRtiS3o2Ge5HGQ4nGS5HaQmuAkzauFh+qevpT4jwKWhVcdwJ+MMT9r7zXdTfyPrX2MP2z4A2PSx/CLU37BmPQ2Wpmo+FZdFP5G8InVhcW+L6wnlsUGmceANxM8KeBOaTr3pIaXUzHuZGokkeKAmyK/m/21Doqr/ZRU11HlC1BdF6DKF6DKV091XdP1+mD7/wYTnHaGp3kYke5lRFoCI9OtaURaAiPSExic7NFuMFSr+kzi744jqer5V8G/+PHKH1Plr+LW427lmgnXYBO9cafaUVcNe1ZbBcH+DeCrsKa6SuvBNV/F4a4s2iI2cCeDO/VwYeFOtiZPZNnaXu9MpNaWSA1eqsVLVSiBipCH8lACJXXC3vJa9pTVsqfcmkoP+ZucymEThqV5rIIgzcuI9ARGhguIIakeXHYbdpvgsAn2RpPDZmtYtgn6rWIAitvED1DqK+X+lffzfsH7zBgyg5+d/DOGJw3vwQhVXDHGumHsqzxcGNRVNF1vXFDUVVnLdeHlyLZgXcfncnggcTAkD4GkIZA0GH/CYMpsaRwIpbEnkMJ/fEnkH0pgd0WAPWW1HKjytXpfosNTtSgYBKfdRrLHqnKyqp1cpCY4SQmvN57SvIeX9bmJviGuEz+AMYa/bv8rv1z1SwThrhPv4htHfUOvclTsBOoOFwpNCoiqcAFRAb5yqC62HnirLoLqAy37SYpIyICkIYQSB1PjHkSFLZ0yWxo+WxJ19kT8Di8+W6I1iZdaeyI+WwL1ITvBUIigMQRChmDQmoeMwR8IUemrp6L28FReU09VuJ+ltrgctsMFQqRw8EbWXaQmOBoKkcPbrQLFqU9f95i4T/wRhVWF3PPhPawtWsvZuWfzo5k/It2T3iPvrVSvCPitlknVBxpNRdZ4C423VR3o3LcKp/dwNZQ72RqoJ1I15Uq0etFrJmSgPhjEHwhRFwjhD4TwB0NN1wMhfEFDechLUSCRAwEv+/0J7PMnUEYS5SaJShIJ0TTRR25qR6ZEt4Mktz08t6bERvNEt73FtiS3Q5/FIE4Tf8X//R+1n68j44brcWVnN2wPhoK8sPkFnvj8CdLcaTww6wFOGXlKT4asVOwZc/jbQ8NUCf7qlttaHBc59tCRxRAKWN9maD3PGISAK5U6Zwq1jlSqbSlUSTLlJFEaSqIkmEBl0EllwEFVwE5ZvZ3qgBMfTupw4cOJz7jwYU11ODHhgsRlt5HkcZCR6GJQkovMJDdZSW4yE10MSnYzyOtgkFfI8trJSBC8dmM1/w3VW92KhOqtzzBlmDWGdT+sHYjLxF/85FMc/P3vIRgk5bzzyPzWTXgmTGjY/2Xplyz890K2l2/n8jGXc8eMO/A6vT0ZulIqFLKqrGrLrKmmFGpLO1gua/IkdlcExUnA5qbe5qYeJ4QCiAlgCwWwE8BhgjgIYJfO5746eyJVnuFUJYzgUMJwqhOGU+sdQY13BLWJIwi5U7GL4LALNml0ryS8zWm34bAJDrsNp926se60H15va/+RVkXHZeIHqN+/n9LFSyh/+WVChw6ROGsWmd/+Ft6ZMxER6oJ1PPn5k7yw6QVyUnL4+ck/59isY3v4J1BKdVnAb93nCNRCvc+aB+qgvhYCvvC8ru39AZ+1brNbo8rZnWBzErA5qA3YOBQQDgWE6nqoqhcq/VDph4o6KK8zlPkMNf4AWaaEkXKQbClqmCdK0yq0CuOl0GQ1mgZRaLLYYwZRapKpIJFa3EDXErndJiy6/nhOHZPVrY8wbhN/RLCykrKXXqZ08WKCBw/imTiRzG/dRPI55yB2O6v2r+KeD++hqKaIb03+Ft+Z8h2cNmcP/QRKqf4qFDIEjSEYvuEdCIYwNaWEynZjK9+NVO5GyndjryzAEZ5sgZqW72NzEXClUO9Mwe9Kxe9Ioc6Zgs+Rgs+RjM+eQo09iRp7CjW2JA7ZkjlkS+b8EycxanBqt2KP+8QfEaqro+L11yl9fhH+XbtwZmeTeeMNpF5yCYds9Tz02UO8seMNJmRO4Ben/IKjUo/qkfMqpeKEMVa1VfkuqCg4XMVVW96oyqvxcoXVHLgtV7wMY8/rViia+JsxwSBV775HyR/+gG/DBuwZGWRcczXpV1zB+xWreODjB6gN1PKtyd/ixGEnMjZ9rNb/K6WiIxS0qrVqy1oWDmPOa78L83Zo4m+DMYaaz1ZR8vwfOLTi34jXS/rcyzCXf4Of7nyaFYVWp6E2sTEqZRQTMicwIXMCEwdN1MJAKdWnaeLvBN+XX1Ly/PNU/v1NECH16xcQuuhsdiTVsDG0m82lW9hcspmDtdbAYc0LgwmZExiXMU4LA6VUn6CJvwvq9+yh5I8vUL50Kaa2FgBxu3EOG4Zz+HACg9MpSbVR4K1lq6uUz20F5DtLCdoFQTgq9aiGgmB02mjSPemkulJJdaeS4EjQp4WVUr1CE383BMrKqF27lvo9e6nft4/6vYfnwYPNhgsWIZiZSnW6h6IUw86EQ+zy1lCRCAE7hASCNsBhJ8GVRIInCa87Ca87Ga8nmcSEFJI9qSR5UkhKSCMpIYUUbzpJCWm4klNxOT247W5cdhdOm3bTq5TqWF8cbL3Pc6Snk3zmma3uC9XVEdi373CBsNeaJ+/bx6C9exmzrxRT37J/dghhjTpZ1qkYaiKTC2rchyefx0adx44vwYE/wY4/wUnA66I+wUnQ6yaY6MEkesDrxZmYjCsxGXdiMonOJBKdiXidXrxOL4mOxIb1REd47kzEbXdr4aLUAKWJv5tsbjeuvDxceXmt7jehEIGDBwmWlWECAQgGMYEgBAOYFsvW/oC/jpq6amobpkP4a6sx1YegugYO1ZB4qJakQz7sNT7sB+tw1Phx1tZhb2UQkNb4nFAXnnxOKHfCfqdY21xQ57D2+V02Qm4nxu0Cp90aIN1pB4cDcTgRpxObw4E4w8tOJ+J0YXc6sTlc2Nxu7E4XDufheePJ7nThcnlwOtw4bc6GbzLNl512J267G7fdjdPuxCEOLZCUOkKa+KNEbDacgwfjHDy4S6/L6Ob5QnV1hKqqCFZVEaquDi9XW8u1tRhfLaGaWgI11fgPVVFfU02g5hDBmkMEa2owPh+myof46pA6PzZfPfZ6H+DrZkSdjBsI2axqsKANfDaotjXd5ndAvQPq7RBwCAGnjaDTTqhhcmCcdozLCS4nxuVEXC5wWwUQHg/i9WD3JCBeLw5vInavF6c3CUdiMi53Am6Hp6GAiVSpReYOmwOHOLDb7NjFjtPmxG6z67gOqt/SxD9A2NxubG43jkGDeuw9TTBoFQiBwOGpvh5TXw+R5ch2f3i53m99gwnvD9XXEwz4Cdb7CdT7G5Yjkwn4kUA9EqjHVl8fXg4QCgZwBKzXu/1+qKsHvx/89Uh9APEFsPkD2OqD2OrrsNeHrCnU9XtWQbG+7US+AUW+DdU5Bb/TKoRCEp5sh+/XhGyAzQY2G8YmGLutYR2bDew2TGQe3heyy+F9djvYBOx2jN2GhOfY7WCz1rHbw9+oDn+Lsrnc2F0ubC4PDrcHh8uNzeXG6XTjtLtw2p24bNa3JofN+oYU+c/63+oDxoat4dtTZFvjOVgt1+xiFXINyzZrHplsYmsoCFvbpt/S+h5N/KpNYrcjiYmxDqNLrELIj/H7CdX5rW86Ph+hmhpMbS31h6qpP1RlzWusudTWYKupwVVziFBtLaHaWqj1gc8Hvjqrk7FgyJqHQhAySDBo9U0cCiEhg4RCEApgCxprfyiELQbtJurtVmOCgM2aB+1WwRYSMOGp+XJ7+6xlwRBeDs/h8HEGIHx88+1N3sdmFXpGBGzSsI5ECk0BsQpRaVSA2sQqoMRmQ8TWMLe2W+u2ZnMRGzabPVzACTYTnkQQAzZjFW12E14PF3XWcVZBaM3BOBzgcoLTgXE6wOkMr4e3uRyI09WwTSL7XFY1KA4HhAt047Bby45w4d6o757GhWPj7WPTx5LmSevRvxNN/GpAEYcDcTjA6yXWY0AZY6x7O6GQ9Q0oMg8GMcGgta/R3AQCEAo1uv8TwgQOf7sK+f0Q/sZl6usJ+v0E/XUE/T4CdXUE/XW46n2EwttD9X7rNaGQ1Y1AKIQJGTDhgsw0Wg8XYpHjIgWcMdY2E95nTKPjIusmcqxV6FnDf5mm72lM+JwGCZ9TguF5KLLNmkvDHGzd+AbXHSEaFViNCj4ARxAcnbuF1uVzhsIFdMhmFdaN55Gqzq333snM82/s0XNr4lcqSkTEuhkO4HLFOpx+zUQKj2aTsXa23G7tsNbFhliDCjd8ewmJIUjIqrYjSCgUImiChIw1D5rD2wyGUDBcnen3W9WZkeVw9WPjbdZ6eLm+HoJBCFgF/OFl6yKAYBB7MIQ9EMDZaBvBEBJ+zbDhk3v889TEr5Tq80Sk1YFQ9M5B92izBKWUijOa+JVSKs5o4ldKqTgTk8QvIueJyJcisl1EFsYiBqWUile9nvhFxA48BZwPTACuEJEJ7b9KKaVUT4nFFf8JwHZjzFfGGD/wEjAnBnEopVRcikXiHwEUNFovDG9rQkTmi8hqEVldXFzca8EppdRA12dv7hpjnjXGzDDGzMjKyop1OEopNWDE4gGuPUB2o/WR4W1tWrNmzUER2dXN8w0CDnZ4VOxofEdG4zsyGt+R68sx5ra2sddH4BIRB7ANOBMr4a8CrjTGbIrS+Va3NgJNX6HxHRmN78hofEeuP8TYXK9f8RtjAiLyfeAtwA4silbSV0op1VJM+uoxxrwJvBmLcyulVLzrszd3e9CzsQ6gAxrfkdH4jozGd+T6Q4xN9Hodv1JKqdiKhyt+pZRSjWjiV0qpODNgEn9HHb+JiFtEXg7v/1RE8noxtmwRWS4im0Vkk4jc2soxp4lIhYisC08/7q34wuffKSIbwude3cp+EZHHw5/fFyJyXC/GNrbR57JORCpF5LZmx/Tq5ycii0SkSEQ2NtqWISLviEh+eJ7exmuvCx+TLyLX9WJ8D4vI1vDvb5mItDqQa0d/C1GM7z4R2dPod3hBG6+NeiePbcT3cqPYdorIujZeG/XP74iZ8LiZ/XnCaha6AzgKcAHrgQnNjvke8Ex4+ZvAy70Y3zDguPByMtZzDM3jOw34Www/w53AoHb2XwD8A2vQo5nApzH8Xe8HcmP5+QFfA44DNjba9itgYXh5IfDLVl6XAXwVnqeHl9N7Kb5zAEd4+ZetxdeZv4Uoxncf8D+d+P23+289WvE12/9r4Mex+vyOdBooV/yd6fhtDvBCeHkpcKZIK2O5RYExZp8xZm14uQrYQiv9E/Vxc4DFxvIJkCYiw2IQx5nADmNMd5/k7hHGmBVAabPNjf/GXgAubuWl5wLvGGNKjTFlwDvAeb0RnzHmbWNMILz6CdZT8zHRxufXGb3SyWN78YXzxuXAn3v6vL1loCT+znT81nBM+I+/AsjslegaCVcxTQM+bWX3SSKyXkT+ISITezUwMMDbIrJGROa3sr9Tnev1gm/S9j+4WH5+AEOMMfvCy/uBIa0c01c+xxuxvsG1pqO/hWj6frgqalEbVWV94fM7BThgjMlvY38sP79OGSiJv18QkSTgNeA2Y0xls91rsaovpgBPAH/t5fBONsYchzVOwgIR+Vovn79DIuICLgJebWV3rD+/Joz1nb9PtpUWkXuAAPBiG4fE6m/haWA0MBXYh1Wd0hddQftX+33+39JASfyd6fit4Zhwf0GpQEmvRGed04mV9F80xvyl+X5jTKUxpjq8/CbgFJFBvRWfMWZPeF4ELMP6St1YlzvXi4LzgbXGmAPNd8T68ws7EKn+Cs+LWjkmpp+jiFwPXAhcFS6cWujE30JUGGMOGGOCxpgQ8Fwb54315+cALgVebuuYWH1+XTFQEv8q4BgRGRW+Kvwm8EazY94AIi0oLgPeb+sPv6eF6wSfB7YYYx5t45ihkXsOInIC1u+mVwomEUkUkeTIMtZNwI3NDnsDuDbcumcmUNGoWqO3tHmlFcvPr5HGf2PXAa+3csxbwDkikh6uyjgnvC3qROQ84P8BFxljOA0ohwAAAr1JREFUato4pjN/C9GKr/E9o0vaOG9n/q1H01nAVmNMYWs7Y/n5dUms7y731ITV6mQb1h3/e8LbHsD6IwfwYFURbAc+A47qxdhOxvra/wWwLjxdANwM3Bw+5vvAJqxWCp8As3oxvqPC510fjiHy+TWOT7CGzNwBbABm9PLvNxErkac22hazzw+rANoH1GPVM9+Edc/oPSAfeBfICB87A/hDo9feGP473A7c0IvxbceqH4/8DUZauQ0H3mzvb6GX4lsS/tv6AiuZD2seX3i9xb/13ogvvP2Pkb+5Rsf2+ud3pJN22aCUUnFmoFT1KKWU6iRN/EopFWc08SulVJzRxK+UUnFGE79SSsUZTfxKASISbNYDaI/1+igieY17eVQq1mIy5q5SfVCtMWZqrINQqjfoFb9S7Qj3rf6rcP/qn4nI0eHteSLyfrhDsfdEJCe8fUi4r/v14WlW+K3sIvKcWOMxvC0iCTH7oVTc08SvlCWhWVXPvEb7Kowxk4Engd+Gtz0BvGCMORars7PHw9sfB/5lrM7ijsN6ehPg/2/vDlUiiKIADP8HMQiCiEaDZZPVJ/AVDCImMW0Qk/gCvoLF4mtYTIJ2H0BsCrvBYFlEjuFeYVHXVXBd4f5fmTMnDHPL4XBn5kwHOMnMNeAR2JzweqSR/HJXAiLiKTPnP8nfARuZeVsH7T1k5lJE9CkjBZ5r/j4zlyOiB6xk5mDoGquUGfyden4EzGbm8eRXJn1kxy+NlyPinxgMxS/4fE1TZOGXxtsaOl7X+IoyGRJgB7is8QXQBYiImYhY+KublL7LrkMq5t79PPs8M99e6VyMiBtK175dc/vAWUQcAj1gt+YPgNOI2KN09l3KlEfp33CPX/pC3eNfz8z+tO9F+i1u9UhSY+z4JakxdvyS1BgLvyQ1xsIvSY2x8EtSYyz8ktSYV+17Dg6wWw8eAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, dense_layer_call_fn, dense_layer_call_and_return_conditional_losses while saving (showing 5 of 11). These functions will not be directly callable after loading.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eNaLU7Pq5zLu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}