experiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "from ridgeplot import ridgeplot\n",
    "from scipy.stats import norm\n",
    "import pandas as pd\n",
    "import random\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from gluformer.attention import *\n",
    "from gluformer.encoder import *\n",
    "from gluformer.decoder import *\n",
    "from gluformer.embed import *\n",
    "from gluformer.model import *\n",
    "from utils.train import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define save path\n",
    "save_path = './trials/trial_exp'\n",
    "if not os.path.exists(save_path):\n",
    "    os.makedirs(save_path)\n",
    "# define cache path\n",
    "cache_path = 'cache/visualize_experiment'\n",
    "if not os.path.exists(cache_path):\n",
    "    os.makedirs(cache_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create data generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate mean with multi-modal behavior\n",
    "random.seed(0)\n",
    "def generate_mean(off, modes, lens, betas):\n",
    "    mean = np.zeros(sum(lens))\n",
    "    mean[:lens[0]] = off + \\\n",
    "                    np.sin((2*np.pi / 3) * np.arange(0, lens[0], 1)) * betas[0, 0] + \\\n",
    "                    np.cos((2*np.pi / 7) * np.arange(0, lens[0], 1)) * betas[0, 1]\n",
    "    state = random.randint(0, 1)\n",
    "    mean[lens[0]:sum(lens)] = off + modes[state] + \\\n",
    "                            np.sin((2*np.pi / 3) * np.arange(lens[0], sum(lens), 1)) * betas[1+state, 0] + \\\n",
    "                            np.cos((2*np.pi / 7) * np.arange(lens[0], sum(lens), 1)) * betas[1+state, 1]\n",
    "    \n",
    "    return mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate the covariance matrix and sample from the Gaussian distribution \\\n",
    "# with specified mean and covariance\n",
    "lens = [5, 9]\n",
    "modes = [-10, 10]\n",
    "off = 0\n",
    "betas = np.array([[0.5, 0.5], [0.3, 0.7], [0.7, 0.3]])\n",
    "\n",
    "cov = np.zeros((sum(lens), sum(lens)))\n",
    "row = np.array([1] + [1 / abs(i) for i in range(1, sum(lens))])\n",
    "for i in range(sum(lens)):\n",
    "    cov[i, :(i+1)] = np.flip(row[:(i+1)])\n",
    "    cov[i, (i+1):] = row[1:(sum(lens)-i)]\n",
    "np.fill_diagonal(cov, 2)\n",
    "\n",
    "train_samples = 2000\n",
    "val_samples = 100\n",
    "\n",
    "train_data = []\n",
    "val_data = []\n",
    "test_data = []\n",
    "for i in range(train_samples):\n",
    "    if i < val_samples:\n",
    "        val_data.append([0,\n",
    "                        np.random.multivariate_normal(generate_mean(off, modes, lens, betas), cov).reshape((-1, 1)), \n",
    "                        np.arange(sum(lens), dtype=np.float32).reshape((-1, 1))])\n",
    "        test_data.append([0,\n",
    "                        np.random.multivariate_normal(generate_mean(off, modes, lens, betas), cov).reshape((-1, 1)), \n",
    "                        np.arange(sum(lens), dtype=np.float32).reshape((-1, 1))])\n",
    "\n",
    "    train_data.append([0,\n",
    "                    np.random.multivariate_normal(generate_mean(off, modes, lens, betas), cov).reshape((-1, 1)), \n",
    "                    np.arange(sum(lens), dtype=np.float32).reshape((-1, 1))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scale the data to [0, 1] using the min and max of the training data\n",
    "maxx = np.max([train_data[i][1].max() for i in range(len(train_data))])\n",
    "minn = np.min([train_data[i][1].min() for i in range(len(train_data))])\n",
    "\n",
    "train_data = [[data[0],\n",
    "              (data[1] - minn) / (maxx - minn),\n",
    "              data[2] / sum(lens)] for data in train_data]\n",
    "\n",
    "val_data = [[data[0],\n",
    "              (data[1] - minn) / (maxx - minn),\n",
    "              data[2] / sum(lens)] for data in val_data]\n",
    "\n",
    "\n",
    "test_data = [[data[0],\n",
    "              (data[1] - minn) / (maxx - minn),\n",
    "              data[2] / sum(lens)] for data in test_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOydd5Qcxbm3nw6T487M5hy0q9Uq5whIICSRgwFjjMEYMDiAwfg65xxwDthgok0yYLIkJARCOafdlTbnvLOTc4fvj9UV5hr82Rgbru8858yZnpnqStP16+q33qoSdF0nS5YsWbL870d8tzOQJUuWLFneGbKCniVLliz/IWQFPUuWLFn+Q8gKepYsWbL8h5AV9CxZsmT5D0F+txL2+Xx6RUXFu5V8lixZsvyv5MCBA+O6rue+2W/vmqBXVFSwf//+dyv5LFmyZPlfiSAIPW/1W9bkkiVLliz/IWQFPUuWLFn+Q8gKepYsWbL8h5AV9CxZsmT5DyEr6FmyZMnyH0JW0LNkyZLlP4SsoGfJkiXLfwhZQc+S5Z8hPATHn4fsMtRvIBbrpK/vAeLx7nc7K/+neNcmFmXJ8r+e48/Ds5+ARADmXQvn/hhE6R1NQld1EEEQhHc03n8FmpZidHQjA4OPEgzuAUBol8jPv4DKio9jtVa+yzn8+wiPjXJo4/OYLFbsXh92jxeHx4vd48Nktb7b2fubZAU9S5Z/lHQMNn4BDtwPhbNgxmWw93eTwn7J3SCb/qnodU0n2RYgvneYxPEJjKUOXOdWYipzvjP5f4eJx7sYGHyUoaGnyGQmMJtLqa76DD7fSoaGnqR/4I8MDz9Dfv55VFR8DLttyrud5bckFY/z5He/SmBoAF3T/up3g9lyUty9OE6Kvd3zl6Lvxep0IYhvbvzQFY3ovmEMeVbM1e53PP/Cu7Vj0fz58/Xs1P8s/+sYOgJPfATd346y8BpGps9D1VOU9scQN30Vqs6AK/4IJvs/HLUykSS2f5j4gRHUUBrRJmOZ5iNx3I8WzWCZ6cO1thLZY37ny/UPomlpxsY2MTD4CIHALgRBwuc7i+KiK/F4liEIrwtaOj1Ob+/v6R/4A6qaIC9vHZUVn8Bur3sXS/DXaJrKMz/8Ft1HDvK+L36Twtp6YgE/Ef840Qk/kQk/0Qk/Uf84kYCfqN9PNOD/K+EXJRm7x4M9x4vd68Ph8WBz5CANgtilYNHtOCsKKPzk/LeVT0EQDui6/qYnZwU9y388A9EB0mqaEnsJBsnw9iLRNJQdP0B65QcoJjMt0/IZsYVP/dwkLcUcms1HDn2bgG86HRc/RLm3iHyj/DfNJbqikWjyE9s3TKo9CAKYpuRgW5CPpd6LIItoKYXIawNEX+tH13TsS4twrixFtL7NsvwTxOM9DA4+xuDQE2QyfszmYoqKrqCo8DJMprxT4cLjo/Q1HaNq3kIsdgcA6fQEfX330tf/EKoaJTd3LZUVH8fhmPZvL8ebse2RB9j79J848yMfY/bZ5/xd52iaSjwUOin446cE/79vAJHxcaLjYyhq5g3nzVqxjrM+8fG3lc+soGf5P8dwbJiN3RtZ37WeJn8TAKIgUmgrpNxZTqmjlHJn+anjNxN7VY0TDO4j1L8B77aHcY0HGfUaOV5fQJtpDq9mptFnmM0V1hPUhX5Bs7yCxtG5/Kzpm/RaCnj/jDsJ2gqotpiotpqospqotpiospqpCCsIh8aIHxpFiytIbhO2+flY5+cju9+8B66GUoQ29RA/MIJglnGuKsO+pBBB/tf6NmhahvHxlxkYeISJwPbJ3rh3FcXFV+LxLEcQJscNdF1n4HgTBzc8S/ve3ei6hisvn/Nv/wL5ldWn4stkgvT13U9f//0oSgSf7ywqKz6B0znjX1qOv8XxHVt58ec/ZOZZa1lw2Qc5dOgQ8+fPx+FwvK34Ur1hgs92kOmPous6iphBrzWg1ol0dj5AWd2Z1C94/9uKOyvoWf5PMJGcYFP3Jl7sepGDowcBmOaeyrIOA87xOENekSGHwqApwYAQJKonT50rCiJFtiKKrG58so5bn8Cu9DElEmNFWwSLDm1zzuXHeVfwbLQIn8nEx0pyucDaTCraiKYm6On9Lfn5F5Ann4fniQ+RNDr4/aq72SMX0RFPMRFNceZQhosGMkwPaWQEOFpipnuqE7naTbXNTJXVRLnFiOktbLAA6aEYoRc7SbUFkTxmXGsrsMzwveMDp4lEP4ODjzI49ATp9BgmUyHFRVdQWHQZZlPBqXCZdIoT27dyaP2zjPV2Y7Y7yF+2km2uAmo3PYESjXLWDR+n4fQz3xB/JhOmr/8B+vruRVHCeL0rqaz8JC7nrHe0HP8/RjrbefQr/0V+9RQu/vzXuP+BBxkaGsJkMrFq1SoWLFiA+Df+j/9G13Vi+0cIb+5BC6UBEJ1GnCtLsS0sJK2McmLPtRQf3gczrsC38ldvK79ZQc/yH0skHeHl3pfZ0LWB3UO7UXWVKlcV6yrXsUqrRfzc98n0D2Cuq0OZmEAZHwdFQQciFhj2wFCOwFCuyJBPZ9gDww6BhOF1cZR0MAsuolIhkrWU5QW1nOVMI01sItVhR4vmklO7GYeriHi8i4KCi5nmvgrhj5eh6xqZVQ8Q7confnQMMhpJr4m2qU62lZpo1hQ6EinG0sqp9ESgzGKkymKixmrmg0Veam1/3WtPtgYIvdhJZjiOscyB69wqTOX/3MCppimM+19mcOBR/BPbAAGfbyXFRVfi9Z52qjcOk2aVwy+9yLGXN5KMRsgtq2D22vPYWzmdb/T6SWsatXqGj257iuHjjcxavY4zrrkR2fDGJyFFidDf/xA9vb9HUYJ4PCuoqrwFl2vuP1WWv4dYMMAfPv8pREniqu/8hK07drJnzx7WrVtHS0sLnZ2dFBYWct5551FcXPymcagphfBLPcT3DaOnJ+3phmI7rvOqMFe6AAgHDuJ/4f2UdY4iiEbEdT+Eede8rTxnBT3LfxQJJcHW/q1s6NrAtv5tpLU0xfZi1lWuY23FWqa4pxD60xOMfPvbSG43hXf+EGHKFDKpFLFIP/6eHYR6DxEdakMJxdFjImLSgpixo6dkbEKYyopWAk6V3VEPh9NO/A6NgEMj4FBJG15vM4Iu4ouVcPaJSyjI2YC9uB1LbpLcwgamTnwM2/6bELQwfv2ryLPOwrogH2Op469602FFpSOeojOepCORojM++WqLJwGB79WWcEWh56/qQtd04gdGCL3UgxZJY5nhw7WmAtln+cfqNDHA4NBjDA7+iXR6FJOpgKLCyykqugyzuej19P6HWQWgZsFi5qw7H0fNVH6wtwu1JcD5QTCOp+jxSLTU2lke2s+BF/5MYU0d59/+eRxe31/lQVGi9A/8kd7ee8hkJvDkLKOi8pPkuBf8Q2X5e1EyGR7/xucZ6+niym/8kIlkmkcffZRFixaxbt06dF2nqamJDRs2EI1GWbBgAatWrcJimazb9EiM0AtdpNoCoAOigLneg/uCamTX655O40d+hPml72KPKWTKlzC0z4n13EvwnZs1uWT5P0pGzbBzcCcvdr3IK32vkFAS+Cw+1lasZV3lOmb4ZiAIAlosxtBXv0b4+eexLVtGzle+xOM/+TrBobG/IxWduZ5BVuR2kdAMPDM2k+5MAU6jjJUUWsaGIswibi4hZJ4gIjYSE5s5VD2KSXVzQeNNGAM70JQuRIOK3W6izFzCab5NmDU/+vt+jzjtwn+o3COpDB9r7mFHMMr7Czx8p7YEq/TXj/9aWiX6Wj+R1/rRVR374kIcq8qQbG89cKppCn7/qwwMPoLfvxUAr/f0k73xMxDF172a38ysMuPMNcxauRZj2Mjg0VHCzX5yExoJdH5iUXgxkeBig4nbM0YSFpl4/giv7fwjssnEebf+F2XT39y0oqpx+gceprf3btLpcdzuRVRWfpIc9+J3zKyk6zob7/oZTa9u5vzbP0/+1OncdddduN1uPvKRjyDLr5c9mUyyZcsW9u3bh9VqZeXUJRS3mFDHJ012gknCtrgA51nliAYJVU2SSPSQCDSiv/w1crt7SZskOisLSD2sYGwTkT62lLpP3vu28p4V9CzvaeKqxr39Y5RbTMx1WikyGRAEAVVT2Teyjw1dG9jUs4lwOozT6GR1+WrOqTyHefnzkP5iIk+ytZWBT91Gurub3E9+Atd11/HI125mvGeYokUBnJ5yXO6puD3TsbtqMJosyCYTstGIlA6S2PBp8gd3ssG7jJ/O+hLX1tSwVNnN0ddeY7ixgVSwHEFUUFI9KPGtGMwa0y+8lsNahD8Gv4Ut6eL8pk9R0r8bPbOTiNdAVDFhQuXi0iYKLBH2aqcRqzyX/Koa8iur8ZaUIUpvnIyU1jQOh+PsDsU4Eomz3G1nJJ3hZz2j1NrM/K6hgro3McEAqOE04c09xPYNI5ikyYHTpUV/NXCaSPTS2Hgr4chRjMY8ioouo6jwCiyWN5oV3sysMuv0dZS5G1A7YqTag+gZjYQER3INmGvc/PT4MB0TMRZWeNjTNcH8MhcXxwVO82tEUn52Bp8hHB9n+fuvYcGFl76lSKtqgoHBR+np+R3p9Chu14JJYc9Z+k8L+8EXn+GVB+5m8aVXsvjS93P//fczMjLCRz/6UbxeL+lEAkESkSQZQRTRohk6Nh5l07HXGBPCFGpuFjnz8CyV0YqCJJI9JOLdxBPdpJJD5I2nqe2IYkzr9BfbGK2ci+VnfvSOEM+sXUx84Tq+9f7L3lbes4Ke5T3NU837mPnchzFraVRB4IRRYrtF4jUzTIhg1WGVamCdamQJJgyCBIIEgjj5EiWCxyIMbxxDNIkUX1KKtcLB8/smaB0yc0bDMHPmLkcsmAX5DZBXD6ZJ7wVV19mz90mmbf40RjXBr+tvpXLZ9cwM7eTIlibGW2agph2IcoJMvI9MbDOCqOIqXErYs4h759kZM4tc0HmUA9JPcSdyObfxkxQE+phx5LdIaorxQhN9VifL6rupsIfYNlbJ3vESAGRZwpPrglwfw94CjnnL2O6uJH5yclK+FmdEtNJgM3FZgZdf9o4SUzV+UFfCZQV/bYLRNJV0PEGsd5yJze1EO8dQrRqG6U7IlUgnEkyMH2Vk8FW0jIjTORtfwQzsnlzsHg82twebO4fgyBBHN68/ZVapbJhHfdlS3AEPynAcANFtYm++gfvtKo5qNyvTBn7wQjN2k4GfvX82S6u9fP25Zu7f2U1uXQ4UmHlYd2E6NMiu9qfpj7dQUT6LtZ+8HVup9y2vD1VNMTj0OD09d5FKDeNyzqGy8pN4PKe9LWHvPnqIp77zVarnL+KC2z/PK6++ymuvvcYll1yCMtDN7icfJREJv+EcQQBBFBBEHV0ARZNBEJENaQzGFKIsIslGDIKMORHFkI6jG0DLrcRoriB09DDJTIao2YSoa8SrpvLl7/7oH877ZF6ygp7lPcwfH/0s8zvu5cnyOWxW/YyRRtYFalUrszI2pikmvKJEjizglgSckoBdBFHX0NIKwxvHCDVGsZYa8a1xopt09naqHBr0Mj3fz/S8KLn6OEY1fipN3VXKoKOKSHiYqeEW+qxlNC34MfJQiu4jUVKh/96DV0NNd5KJbwI9iWicht17OiN1OdxVa0bXk6COoxrL8ISOIId+glUtYPro7RQFRVZGfoSnowtjtwiiTu6yEL7iOAdGi9gcryVudpISDDgjIUyZ1GTeRAGLFUrNYcrMIY5aytjoXkZSk5lCikw6QyQWp1xQKUNDScRJJ+Kk4nEyqeSb1PD/QNCRjQIWuweQiAUDaKrypkFNRhtOKQeb5MYi27Hn+nBVFZKoKuCrqQzNkplbygsZPjTGkwcHWFLl5WdXzibPMfkEoes6X36mkT/s7sU8xYW9PocNc2uxdofZ+8jjHDzxInZDDivnX0PxyplYGrxv6YapaSkGh56kp/s3JFODCILxpKALJycy/fexcPJY/Ivjye91TSceDiOIIjZnDhMTXg4enEOutQvDSD+RfiuSUcVbF0O0JEET0DXQNQE9Y0BJiGSSApmUkZSSQ0Y3IaHjMBpwEsaQGEbRBCKyCUXwkkmkUdJpEAR0QEdAQCe3soZrvvfT//9/9WZ/X1bQs7xXGUql2f6rFXzTEwdESsN11EcWUp+eg8XoICULRCQdPxp9uoIfjSg6KTSKw6PcsPl3FAaGeLrhbO6tORNVEKmJdbBu9CVO2KZwsHIdXruJtpEwSz1xPj01hT5+AsPwHurTRzCQIaOZaEmezrH4OUwo5RiEBB65G0ntIxBvYyKRwFhdifX0GQxZx3na72TAdRpysgXX+M8RtCgR782kbItxTzyNHP0ziqmGYO5nQDSTk4lRpreSP5SkuHmQM9Q9zPW0kOqCnT3ljDispE0Gcsur8BQUIskGooEJRrs6SMaip+pKB9IGE2mjCdFkJGiwYLBYmZ3rxeuwY7RYMVltGC1WjFbL68dmC4mODnrCd6K4e8lPXMzUpV/ClDfpgREaHeHAC0/T9OrLpJNxrCYXOXIeJsFKSkiQkpMklSjxWAhNU9/w//kNOWwoWMuE5GKl0Mla6wgWmw2jxYLF4WT22vNwePP44tONPLK3F73awYL5RTw6qxpZFOjeu58XfvUjlHSahb51lOdNxzovH9uCAgy5b75uiqalGR55lnisA/2kTKLrp451XTtZWzqT+qaDrqEqGdr37ULJpKmZv4hUWuTlFwTk4SHEcATRoFM01YARieGuDJoqgSYiiDKCQULTdDRFQdNUNFWb9GAy2/AVmjhL3Ek8pnE0WMhA3InOW5iRRJGWqunIJeX89uaPvq02kxX0LO9Z7m9vZfDFs3nC4eDS9osxOmcTMxfgT+mMp9L40woTikpQV/nLPuQZfQe55fATZCQD983+IAPeqTjRqPJtwN3YgWC1YbBej8UqETC3ENHSHFWdtGkuPq1v5UPmhxlXKnk1dSuBeC6aZsBhGqTKuJMq82YGbUGOmowcMZk4Yjbjl0Q0wUzM+zGS1jnIqU5ywru4o7yYi2Z/mJ8c+AX3B8tIWmaSM/xt5NQJXJSSk/4sCYOZ0fw0IbNMSnjd+yQnE6Iy0E9B+wh5sRFsiRBaMII1EcZqd1DWMJOCmilUz12ILceLUYvTf+w5vj6q87xzLr50gKRsRRWN/Ki+nEvfxAQDMDT0FCdavoIkmqlQPov0ajG6otHoOMqB4VcxjMaQNJES6xSmOOdhNtgZyfQwnOhkLN5HRkmDrpORZF5btIa+ogoqe1vJaRlkh3sJRi3D2WObKU0OvGn6uRXV1K9YxcOhAv58bJxMtYPrz6jm61Mm7fUR/zjP/eS7DLW10FB1GtNYjKgLmKpc2BYWYJnuO9Vr1zUNNRBAGR1FtNkwlJb+XWaXv5zWv+rKGxk6dIKm7sPosQCyaKDOvZAK6zQO+7cwmOjAaykmp6gIQ74dySQjihKCKCKI4qRPuiCQCo4g9O5ifDzKcNIOCOiihGQyISqZUz1zRTbQW1jOUG4Je+achlHJcMnACe684SN/Vxv5n2QFPct7lu899XNenfgNgWQlI/1vnArtMUKJz06p10GR20yR20KRVaL4oV8jvfgMxpmzsXzuW3TIdtomRrBNfIHhZxVSgpXD0z+GRVMojh9G1NMoogWXNsqF+ibMaRfb4lcQTNWhCwpqyUH6RT8Dcpgx5wlGjAH0k0/9ZUY3swQrloSbe0s+RUIyk59McF/z7czWxxDyGggVLeSl+o+wLZzimeFh0hjJHfo2KK0YNJnZ0TOpaV+IzxLDvvJe9KLribvPoa3rAG3+IdqtlYwZ3afKbVDSFEQmcIUCOMeHmOUf4Kr3XU7N/EWTAXSd7e0H+XJPiOOSB2cmQtjg4CphkG/NnYfFmQ+AosRoaf0qw8N/xu1exJSp36Nxoo9ndjzMwYmjjJliGDQjGSlOcSqXMjWPAlwUyV5KpXyMsglJNiAbDAwZbfzQUUyvaORyJULkRIwtgxqzfDJfWuIi125kzD9Ba1sbXV1daJqKOZNCHOpF0Cd9s3VgZ9WFHNSLyNQ4+el5DafGAVQlwyv33sWRlzdSWFTGEt8sODGE6h9Hz4QQxCh6KoTiH4fM69PoRYcDc3095oYGzNOmYW5owFhRDgiowRSZ0TjKaJxdmx/nWMsWprqXkMiE6Ik1o4si+e5pLMs9m/FgD3vH16PoaZZdejXzLr7orwarY8EA3UcO0nX4AD2H9pBMpBDQcTtTqI4MWqaA6GhsMrBspDie5uULruK58qmn3BqnJ1Jc29lNw5QK5qxZ9LbaTFbQs7wn6UmkeOWu93Gnp5vU4IVcZaljZn01mb4T+A/vIj42BIJAcV09NfMXU1FYRvjb3yF1/DjeG64n99ZbEWSZVGqU/fs+zOGH06gxBxd+/U72Do3RtHkjisnMgCRTKwQ5e7SPI7HltFsyjDg6GHS1MmobIC1P2p0NioAvZGaWbyZrll7GvJKFmBULt23cz9M5NgCWjHbymfxWHHGBoeYDZAYGMCWTmOrs3DLnTho8HlpiSQaTaWZ23kO//NpkYxagKFzDtImZnDHrJSpK3ses+Z+GI4/B0zczUbSI5plf4tCRw7SNhOlz5tFTWMywNw9NFCkd6OSCVIBbLr4Il2dyAFHRdB7sG+T7XcOENQFdEKmNdXNvYjP59XNojDxCX6SHMesqjqdk9gzsIamnEFWoG65m8cAHMeFGmTtMY8lrNE80EU5PDgbKgkxNTg0N3gaShnKeCLoxmcr5QkER9z/fQvtYlFtWTeHGZaU0HjvK/v37GRsbw2QyMXv2bObPn4/T6eSxPzzE4I5XsAf9eG1OzDocjbpQNRMeQ5wl6XFyMhn0iQm0cJj+HDuNJbkYVI253cN4dAnRmgOiA8HkQi7Ix9JQgWVWJVosQrKxmcTRRlLtrZCZnJ2JwYzkKkV0liG5yxgxKuyI78AiO0moEURRJpnjo7hsLism6jga2kp78CC+sgrOveUz+ErLSaRVTBIMtp2g+/ABug4fYLSrAwCrCYoM44gWG4OqRHTCCICvtJJ8VyGp3Ts4PmMmcbOJV+rm0plbiqTB2oMxZnanEYC5a8tZctHryyH8I2QFPct7kp93DyH9eQV3eSwUxs3YtASyIGK12TAZTJMzOtNp9GQKKZ3GklEQERBtNswOJ0aDAVFXUJJDKBEQ0gJuXx6aopKKxTAZjKQ1Ox0DCmFbhrEcjYAjgS5MXvOWtAdb0kNxykluykOO4mRqbR0zZs+hurqanQeO8LGQyIRJwhcOs7b9AJqSwNnTjN0fnLSSijpoArNzBphRN0b4sgfIzT+d9x3uoC2e5Lz4BraO/oHKZDExMcmo0Y9RMTPTYOR0+2I+cM73Mfa+DI9/CFwlcPXT6K4SgnteYuSpuwjs7WPj9DN4atVaRnN8eEITXO6QuX3FEpzGSR/ziYzC9zuHeHDQD1qSRck/UJx4mZakyIg6+ajhzlgoGJSoTZdQ4J+FoC3C6jSSW+ait2mC/EonKz84lYQzSLO/mWZ/M43+Jg6ONZFRIgCISKipfKRMKedVz6Q4LjPeOALxBD6LiWKjATEWJTI+SiASIShJFEYieAJhciNx3LHkqR11VEEkYHLgd7mxpwPILifOKbXkzZmL6vXw0oaniYaCrLzmRmadfQ5aNEPwQD/DBzoIhCeIWpLkWn3kjTgRVNA1BS0yjJ4eRI/3o4x3kRnqJCHotOfnMOBxIAA1OXlE0iIRh5d50mwORzcTCo8y//xLWHr5B/EPD3H/Q3/C2LkTMZFBVXUEAQrz7OSbo6jhYQbjDsZTkzd4tytBri0XB16sYwEcB4+gWq28cOVZPFl9Gn2uQooDo3xi8BjnFRuwuCxYHCak4hlQNOdttZusoGd5T/KJF54k1P95OiULYYPGomQGVddQAE0Q0UQRRRAmv9N1NEFAEQQUQBVARUAVQeXksTB5rL2JTdWoSviCBnInTJQzkzPmnMXg7i0cH46yo3g1oqix2JvEnR4nqGoczS8jo6QpGeqhcqAbe3QCAEHWseXFsRapDLlT5KYD5PVUcaJH5rySZgoLgjw5vYH8qV/mu6OlDKYyXG3azpMnfsOZgSWcFVrIk67tHHYeRZEy5GkOLq66hg8UVeJ54vrJZXev/jPkTi4tG4/0cvzR69BeHuCAtJAnz1jHsZqpmNMp3uex8YnpUxAzI2wf2M76ni00ju5F0XUkBBxSCYtbR7H0eylUZOYuW8rhY2UoSjH5lSbO/fgCzDYDbftH2PZoG+mkwvxzKpi7tpyeVIYbmrpoCse5xhImcGg3A/0nyNf82FIxrAkRR9yAgI0Jt48Jd+7Jdx8B1+S7JknMO7GfFdtfwKBkAAGDIKAqGWxTa3ixbAXHhsZwliqcPbGX0fE+UgYVwW1B8jkJJCYIZcLodiNJg0pS/WsPnjzRxyLnfJYUL2FJzXLSo34OvPAMrXt2TO4ipU8OT9Z48qkZjKB1tiApkyYbRRDwux0oDVOJuB0MTfhJxCZvXnY5Rak1iMcUJ6Ea6Ip6CKStgE6RJUytc5wpDj9Ow6RnUrjXzMCuHMzuDEeumM4Xpt1GXLTwya4/Yh9IMUQhFfRxLi+TS4BgzY24P/jDt9VusoKe5T1HSyzJq7+7hV/l7CRXETChc8XCzzDFNoXQzmaaXnkZuwZL+kYxDgyinnsuqcsvI6PrxMMhQgMvYyt6hvFGF0N781DNFlLldaiiRElJKzmWUTrW56LpCu76asR4NfGoFTW2FaJDAAgGI87p87FWVLG3J05XzyiFwghFsSFsiUnvEtVgRjLZwKPgKglRMc2DLXc+O3fezwcn+shX0ii6wGMDi/CnLVxVthuLPcWu6S42SfU8b/s6acHMlcYtPHXi96wKnsEdQ5cxSpTHzIc4XLSZIdM4EjKn587h0rYdLE2mka96AkrmTeZBTdHa+jWGWx7He6SK5hYvjywopblUwZA6hqSMAJBnEJlqSlPhW83diYsJm3Ko7Wzi5ugRlgWP8sLgDag4qXJtpq50E2JsmJQAkfKl2Aqu4OgOG/4+M4IxRFjfginSAwkVFZGww8WEOxe/20fI6WXC7WPck0vU5nr9T9UVJGUEOTOIOzGEXTXQkbsGe2qQWe33UdQXxDtmwBGfnIUZNSsM+pIM+RIMepOkTBoiImZVxpDQMaVFLLoROaHhMjqZu/RsCryluMwunEYn/ZF+dg3uYs/wHiLpSSH2hIwUT1gpGjeT6zdiQOLcD34aV6udo+Ot7NKOkD8+hDY2jGoycNJIgymj4Ikm8MYSaJKBDnceGaOKJmkIuo5Pj2M3+cgrDiKburC55mItuoAt0Qyh/ftZu3UL/VNq+doNt9Lt8FChK1TuaYFoijsvnsX+Pcc5NtROSlQRdYklZbmsuSG7fG6W/xB+0DWE74lz+Emegqzp1KRVTpgnG7qkSZx+3MI1G8IICOxcPJexoqpT53q9vUyt34a/y8vAZh9anpu4pwJUHXN/B0atDLQMWqYdk3MdOVMqMRfGmTi+i3BHy+QsEVECNYMiGwEd+WSvLWx3MZBfzmBhOWvSVqpL72MgmMf4WDmZjJlqejhb30G+MEpcKyGq3IBICln/MX/omYe7oIjLvRvRxCSHZlhpMhXyHfE7INq4WF7PS51Pcp50IdccXY1NEOhMq+yxH+F4/g6aLP0k5Ch5GlwQjVNe/V/siC3GbpLJ9YQZij9B88RLdKUho4NBFXDF8hnOX0HIvYjKVJLLwkeYv/85PGIYQ54NZ2aCgeBydkU+hK7FmG/9NXNzDhAQbBzyF9A37iSqmtAQScsGQr7ZhHKX4XdZmbAECNhg1JtLxmA8Vf/OdJSyxAjFmSAFyTiemIAtaiIQ6GU01Y2qJbEnVKxJlUPTcjk0+4OoooX6sQeplV/GHrZj7PZiHjBgiumI+uQTVcpTwPLlyyifOYecwiK6jxykdfcOeo4dnuxtCwI18xez4IJLKaypJZWIc+zljRzY8BzdygAjRSr9rgij7iSaCJIqMDNSzuzBMhwRgWB4EPGknV0ymLHmTEFViujQvcwyPo4nOMHxWB6ClERAxxeLUxiO4gsmMGfe6K75P9lXP4MfXP1RAk4XH3rhKa7a+Axxk0ZPnkBPHvTkCvTkCfTnQkYWWNPr40dffuVttZ2soGd5T6HrOu/bsJ2yzmvZbbESlcCXNnDH1G/Sl+rC+8R6pr/aSWeBwI8vEhnNEbAlDdRaq1lXk0te4iUykVyOP5GDkpdHzFGO1RwjVzCQ6p9HKtKNkniVyR0WVYL2AhyxUURdJWVyYMjEkU76U6uCyHBuEUcaFjGWX0zA6cWVFrj2cJg/REJ8sFzjMrUE01gTFvUuXOJxArh4mSU0MZWklCZg9nNLxI4h+jTPDkxj7vKFrEw+hKalODi7khMGla/zLTKqSl3klwxG2rm24jrOeGUx+UmFmKZxJBWnJ6eZHZrGcOFO4pY2dEHAnfQQkEAwTJp8rCk7NUkzS9JJztI1ClP9yGqaZ/LP4u7i93HcXo03HeCqoU2cO9pK0+AaAslq1HQ7waqjzCmKYwol6TyYot3po3X6afjtdsZdbkKOv+xt6+TENFxRBVHOYJINGHQJHUjKAglZQIpP4BrrpWCkl6KRPnwTowjo6AiMefNJmiyUDXYRc3rZe/Yq9vtmsbhnH+cPbaRkVjOSMcXwoTyGDvhotU3BoUQoSo8iaCqy0UTx1GlUzJxDfnUtg63H2fv046QTCQBsOR7SJydSObw+4qEgmqZRu3gZsViMEy17MWkyhpNd8KRBZTQnTcokYWEWBanFFDgLaHTFqB25i54JBzlxP4powF2bz5HKMhx2G+OWmezu1YkEFKzJKEJcw6CqGIqNfG54L9Uvree3N3+KZ6bOxpgKcdrYJlLBw/QbRolaEqeqMyepUxVVKc44EUJGamuX8KEP/+BttZ+soGd5T3EsEmfj7+7kT55Hyc/o9Bkl6pjHQ2d/h4Hbbidx5Ag5V12F+45baYl2srf7NV5tfIJS5wirczI0Bwz4X6hEzKlBs3sQJJ2KnHoibTGUQBdq+giSZCUkGbBkIkgn3eY0QWDEV8RAQTkjzmKMDo2+okr8RjcFo/2M5BZTH9b4fEsrx5VB/pTMQ1AyfNb4AOcKh5kQRX7rdvEnpx1JtVCbrmKJI0GhdRxFlclPZ5g4qtDVnc/yi6uoSGzCEIuSWnIdryRa+Gb6egQ9g3voi0TUDOdWnscZXVdS1BzAI4h0plTu0VM0yyrVxl4WF/+WnYY4BRqsiEVZlkhQrEzeiCZw0K3l06fn0afnMqB6kUMpZF3iYP1cBvMbuHh3DEdSQzaPstSaxKZ5OTa2h87IEfbNX8PWecvR/4YPd9VwhvP2RXHGdVorVUZLouQGh8kZ6sHW34WcmHTRUySJtNuH31dBE8VcVlRL+YRAi1PkCUsXq3c8jykeoX3pYp6fvpaC6CjnbX6BYmMCW10Qu3mA3t1OnpQuYtBcxKXydqoTKulImGRk0pxkdbkpbZhJeGyEobYWrC43VpebwNAAakbBXVCIpiiEx0dP5l7EaKlGUa1MeIfpL1IIGgOM5gQIEwQdqsbymd5qwRPWSUkm7AvO5ESJD7W9iOm9aQSdyTEaERTp5HiNoKMJGTQ9zUiOxstzCghbjUzrHmR2RweCriBpBgTNSLFFpSDRi0dPEXN6eU1OMqrHsBrNnDFvEVeuvOhttZ+soGd5T/HN9gEqH7+G7xePkJ9RCJtm0lf8Oer6e1izfwdXnnMWhSvnMja2mf7uFwn4D5KOCqSjBiLjbsZOuEEyIqgKKCmk/7GXrw6Mu90EXXmEnF7CdjeGwhIa6upZLtmxd23h25LIYes0XGofJiXKqKmetYMZLtq+iS2GzbRMHeNDiQCXR2JkdJnfq2tpNpqpsYo0GedT4d3L/LwdmKQU/eE8rHIcm6QhC0nany8jHTJSe2kXJucbtx5LYyCqWXhyXKMppTFfyqVKK6JBcVGdcqKmbfSjMxiwIyfsVHgfpSanBFGqxR/3MRx0EXftxlX/PImkmVHFR9dgBd29KzFjp0Q1U5YxkUgKZEwijy2105Vn4Kwjh5iz70X8NjsbV13GQH4Roq4zJZ0iNyRz3kAaT1rnoJJhs5Bgru8lkmM7mToqY5ooJJlJA5MVbTLqxDQjYdFGomwaYw1LaUwaKB1JMT+skNJ1/KZxwkKS2e5C7qvJYeGBl5nRvJvhsko2rLqIsNHByp0vMqvlKEmXl6KSQtTQIA8k5tFrLuXKoidYWXeAYNc0gh2FZEIiSnIQJR3lzRBFE4KYh6oMgujA5r0So/EYo5HjJIsrmdPQwIWXXYamqvz5+Uc4/MxTuGNpYiaVpqoQXSUZ6vwXMbd3AbIm4pylklSaiMTHGVHcRBUrBjWJoKroyDRVTaG5ohRbMsOKY2MUBUDUjUi6AVnVMasKsqoj6IAmIOoCov76kga1S9Os/tDat9V+soKe5T2Druusee4Qy7su5xmnmbQokLbdhC+aS/lQF0nZgCsSIDcyjDMawhRJIP6Pa1STZDSDiUFfIZrkpWrMSspkJq6+ij0S54Wlw0y40giIVLhqmJasQAjE6TS20+yZx5jzIkDAEl6Pbp5LylTKzW1JKpyHSATamX58C/NzejGKGrquMyi4+Yx6B70mM5dVbGF6wQEEwDA+g/CRUoyBIp5x93OkoolZgQbOi52gvUPEZhZYd04BJBtR433sdJ7BPnIw52WwKmH2+dtoyyhc6EqzwqEgv0VnOdy7gKF91+DI1cmd9WsMjlbSx6eSbp2DM99LiZ6HOVZIWhM4GFcZVXQcQgyncogOMUZAncCcDrJp+fm01MwAQcAQTaJ1RBFkCV80QnlggtPCoxSnRvAnB0iqk8KZEWRGTHkIBjfllCCJ1UyzbWeJ/QGOCqU8ry5mg7qQMdyn8ivqYGBS/jMCzLXLtM/NxRjs56ytT2NLRnn17AtoKprOwoluFj/9EMZMBkNxOYbpZ/Jgo59hOZ/L9Nc4e/lmRHsIXRdITlQTaKsh3OsiHR0HRES5ElEuxSq5iMceRxeTnHHNLRze+BgjQwMkamaQV5jLwnUNdO7eQ+uGI5iiYYKyk7Anl/iSOIPxhSxtKsSdsNKVc5RdFU8TNvtPlUfWJUrS+VRkirErU3ipcjkDDgdre9Nc2hYnKAuMWUT8ZoGkJUncotBvMdJvMeE3mUmfnKAkaDorJg7xX5330eWZx2U3/PRttaF/WtAFQVgL/AyQgHt0Xf/e//jdBfwBKGPScPkjXdfv+1txZgX9/yb7QjGe/d1j7Mv5IVZNp8XsJCdzCxe/9CgwuXhR1Gon7Mgh5HCTsNvIc4wQN1iQOg3kG0GVjBjqF+HaMYScLsNqiZCfPs7x8e2suvJGKtau4Nj4MY6OHmVf01aOa62kTFWEPR8mYSijPNbKkiE/L1TOQ8XAt4+nuXhNFebQs/DqdyE2xmDCycbBKYRkgbEzFM6ZkksyeYiUYmRr/1JMx2dxVX8MOl5Biw4hly7m2Ow6vlP7KPnREm7qjtE6aCI3R+M+z+V8W/otS6VmXiz9NF0Lr+N7/nFq5Qk8g7fRklA5x+PgcCSGOWXnvyJn40nkkzBECZePkHE/TTJppbDvLBwTMzCFy5GY9EFPyAlU1zAjUi8t3TNRFBMe5wEGunYiS0ZSmsLh6YvYNXclCbNlckAYsEdCnPPK87hHh7ApEWQmTTkxoxOPuYgGQwkGSxFbLBaOhtswJAMYECkXp5OPB1lIsNj+ELNtG9B1gbA2jS5DBQk9Q2lygBx5kAQyX9VvZ2OmHM0oYpnnI2iD+Ye2s+zQqxyatZitC87Gpac57+gu8vdsQRAEojMX8VSkmrBm54LoXj5UW4hk7yDqOUzK1T15naQ9qLFcEBJocorebTaifUZyGiIEj9sRjDqpKXUoqpEK+zbGGnPQYiLjBg9DOWWsK9nP9ilnYDyyhKoRFckUYJ7nMCZ1DsfM3bR6D2ATSymiDFfQiRQyszPHxP2zinGmVT7UlqY6qeGTg7jSCQJhC3HdjsSkSJolBY9ZwSrGCAmQlPopzzxJsXKUCSmPo8U3ccZ1n35bbeifEnRhcs+pVmA10A/sA67Udb35L8J8AXDpuv5ZQRBygRagQNf19JvFCVlB/7/KF0/0MfPhz/Ot6iPUp1Icca9j2TEH09qOMrDqdKRUDpbIIHP17QzOtLHJtpKD+kLSopGcWJiZw+18xXyC40fnMhLOo9jSRH7Gws6xpylyiFwxYxBRT5FR07R4GmguGeFJ8VK2iqvwphN8vhHaXH7urS4hL6HxsxaFxYt6kPd/FyY6Jpfl1TWGp6zivvEeGgqS2HKTZOISgX1u7K3FSKMqpYMdyLpG0mAgLcs4Ewk00cCr087gx3ULUVUXXw7cRTAoUFGiMWRbxAXGXZQpu0hWfIL19TfyiXiA+RawDH2G5vAQ1/oUil31PDgwyPXRWqa1XYsNIzFvI4MzfwOA7dANxMamEjeGSDo6ydj7SU5UExuejiinkMx7ifTtRzSYaS2q4LXFa/Dn5JITDRCw5zCru4slB17BEuhHVtIM+YoZMedTPxrG5KzDaCrBoJmwxbqYYfPgMxcQSk/QogYRyupIm0YZHz+CaXwWKcVOiTHGnMK7KFR3YlD+2/YlMGn4AgSJ1wpr+cXQR9iXKkablUM6z0Jxcx+Xtr3IuAbrV19GyObCmExw5q6N1LUfQUHmidJLCAgu1oxt5v2nz2T+wvPp6jjAzsj9pOxHMUsKGSB1IBe90YPuSiGETAglaTJlZahtEqbwMKRUhkz5NLkbuMrxGl1T6uj0n8vcdgVVUplr3U2RcT5mxU600oRzTSEWi5P2/ROc2D1MTyzCppkpWspLWTg6xPE8Hxl0Zgd+hT+4n+DJDUdcqpvM8CWMhGsRBRVNl5gut/NFw8Ms0U+QkZzEfR/mCW01M2sKWbCm5m21oX9W0JcAX9N1fc3Jz58H0HX9u38R5vNAKfBxoALYBNTquq79VYQnyQr6/z1UXefcJw6xpuda7s3Vsao6I74vc9Ojj1IUCrMq0441N42hSKFpuZW4TaKyWeHp4EXsyptDpy+Pfm8xgq5TOZLm/cFDnN6bz2vdfwQhTVrVKMmVmH16ihPOKE8Z1vEya0AXeF9vio+0p/nBzAyb8nKZP6Hwk7b9FJsewBpqJiOYiOsyUXc9B6wyUm4PZlcGpdeI9JoX89E4OdEEIhD05LO9YRbrFy2luaoWUde57s9/4LJXNmJQNcImK79fXkL7lBhXHjSQzmhUVFjZb5rFNH2A1cIGZGUVj+TexFdnmFkyFkOKfJ82Qw/XeVM0mARCveWIlhA53R8kk7YyQRT77McwOkaYOLGGidazUNM20F/fWUdN95GJPcGoN59XlpxLb0klnpCCUYVhj8zp+w+zYP+fAZW0sZrhnIXsnF9Jf6mVKb2d3P7Ig4gY6XKmUQUFS04RK6afgc/vQ01YMMlNmKSNBIWLIFFJkzhMZ9CNKiVgUR9XLp9Obtd2aHzq5M1RBKsXPTbG4RlOjihTef7oLeyqzUEts+MZSHDLxAkih9fzzIqLaK+YRr6okw4Gmd24i/rjh1jvW8OwKZ+1o5soMfayqaGPoFvljOLTOa34NMQ2Px1/eBbJYAAEis+7lP6+PpSDuxA1lR5LKftdc1hhPcE11s38KPdOph6XsaZ0HAXjrEpEEIRaDIU2LCtL6ZlI0bZvhKGOEDrQudDKUyUicibDrZkwv/EWo2fi3H3sNpYHmjnoMHOkdgoIScqsAQBeHZrOztYV3KBu4zJpKwlM3K2cy1b5QkbrC+nMN3JWSOMPF729PVP/WUF/H7BW1/XrT36+Glik6/on/iKMA3gWmAo4gCt0XX/hTeK6EbgRoKysbF5PT8/bKlCW/51sD0R48q7XGPPcRkgU6bAW41Rv4rIXHmBe1xCFBhOaf7JRaFYQp9VzQrDT73CAQWfZwvNYf8jMiUqV5kqdAUsepkyKuu7jfOq05eS3P0+v8iQvOlfxEueSwcAFAxmuaUuRUiN8eYGFNlcOn2xv5aa+X+EVDxPGxisspUmuJr+ojaKiE5gjKRK787AcMVDQP7l1XU9BMa/OXcRrcxbSl5tPgX+I8sEezDjY764iXOUif2KML/7uRzT09ZOWRAY8RtbPg5qRQnKNUaaUTmGDVI4uKCzgMNOsGr90X8FDU2ZQ23kQMforAs40H1BF5ldF0ZIyetyBWfGi5gyjGaLouowgKPREvKg7v4EaN7Pg/HKG+7dx4LUX2L5wNcemzsWaSHF+8wG2TZnOoMPJmq1PM73lMDHzVGKW+ThkJ7kmC2pc5USZkfVzTCQkgas3PM3lm59jpKKAQ/VzMJkVluqHqDMvJBY+G0GX0cQUe/J384prDtfML2PnI+0Y/S56choxrQxwzfwPUJNKwI6fQuOTkD+DTLSPvQ0CugDRHZfyxeILGahzYAxmuPx4gOWRR3ncXsPWxWvwSQIrgyO0jA9g9gcZGPEybvSxZmwjVfEeIstXsuDSa8h0t9P1s2+DrhP05NPvLaSusxGDqtBlq2Cvax6C1czd4vfY4zmPgZGzyQ2qqEVmVgu9OGLFiKYUiYYyjo/E6T0eQFMimE09KDlBnlqwhKN2H7NbGrnWkuQLJQ3YM1Eeb7qNZE6I8bICBHHSnXQsXEAiWEFwwMPptDNb2Yem6zyirOaezEVYyopprLUj6jCtKUKprZe7b/zg22pH/6ygXwas+R+CvlDX9U/+RZj3AcuA24FqJnvos3RdD79JlEC2h/5/kc809rDgwV/zrWkvMC+ZYmv+VSw/rDO3eR8rRoeoee5Bjm2+FkObirlzFqmDx3FEJmcAapKBkL0Cfcp0as6cj36ik0eNAV4orKK9ZgYJgxGrHiONEQWZ04fjrG0fJBpNIEaLQROIWnQqhQ5qpa3kG9pwGXrok5wcLCwgZM3BeESm6FCA8r5hAFpLK9g+eyEnGmZgVAVyehvJHenBExhD1DVigoeHyi8hIxrBJqLPdJNyWrhw9yt8+E8P4Yon8NvMtBQ6CdoczPP0k28/jcOmBlqlQYxCmmmZE6yvvICnq+ewUhkjFvgpg+kRfjjzNpzRFwkmdyMIRnJ9qykquhSncwmvbPgmovUREuF8+kfOZ+JYB4dLprBn7mmokoFLt61nninAD6avJWRxc+7LfyIeMHLYNZMlM91UxPfQf2ILFlGlOCFBpAqjtIAdM/I5XJVDUXiMWx/4PVNG+xlqqCdVV0u9uhxJsZB0jSOFchEEEceyYjxnlYFBZNv6Jo6tHyFDih0Vf8Y3S2Z58TLmDJ1g6u570ESZJreDHaUqVs1KcUc5DxquZXN9BUJSI/9gPx9TXmIgnODRlVcTtrtwBp6AzHEUeQ22Iy6iGTvnjGykMtFDyGbBEU8i6DrhHB/OUAB0nV5nFdvs8zH78lmQ2M51llf5k/I5igZNJG0SM+pc1HcGEFFI2Xp4ebSCRLwPWW8nJg3QmltAb1EVJ2pmoIkSZ+xaz6zmfQhMrmVutmYwOZIYHBnMDgtR1UMg4kbTHVzsHac29hKCliCurmLY9CGedhVyf5nImEumeDDF6SqMxfwsqXHzsdVL31Y7+neYXF4Avqfr+raTn7cAn9N1fe9bxZsV9P9bpDWNS/54gPP6budXxUFKUnC07Md88qHfUDbmp3aqm+il3ciyi0j4FvbsacQmgrm5hQrjAoyD7RQmujBM9JzczEAgbDGSKHZy5Iwy7ptyNSPO/MkZhcDsTj9nNOo4khInSgWm6YeoGE8wnK4lrk2uVqgJGqoQxONvp3zwKK5QJ11FOYyvWIl19RrK8jxENzxHy9ZNqFqGwtKZJJwFzH/st2ypqUM1pxEsTnKXnMXWhIedYyLhSjdqpQNzIskXHn2AJQe3Ieoae6qLCNgsXFDcTItpOSnxNIJSC32iDbuUZmjlRfwxY+QDeTIdbV9kODbM3WffTZXVgsmYj8HgJDge5YW7DhPsT2Ms30HZ3D+iIvDbzMfZY1vO/GP7ufLQk+wozGXzvDnoBPEMHCcmpbDYxlHEOLqSYWGrzlmHdGb0vHXbV0QRxWDElEpOLkJmsRF0mwkY7KTMHmZ6SyEoIFptmGcWEagWaA+NMXrQjBjJozenny21zxGwByjPZPjB6DjT0hkecdi50+MmJU7andOGWsK5d6BrBoyHAtRnXuTD4/v4zYyP0lI9A2fgGIbobxCUDIme69FSBZw9vp7aSP/JDS0m/cRPePPYb1pN2mClzDXCh+KbGDOtQhyaOrl3Ua2Zc8dVDGmNjPYch0MB2jI2enwGekqq6C2qYsxXCIApk2JuaxPXPf0QgcVwb/WHyYsOc1Z0PYagQiJoJREzo2fUv9rGwiLpmK15JA25nLA7aC/xopqd1CZTHBuS6da9uMUE18ywcdsHznlbbemfFXSZyUHRM4EBJgdFP6DretNfhPkNMKLr+tcEQcgHDjLZQx9/q3izgv5/i83+MM//8iCa7wYazTIDlgYc+lVc+ezvmdM9jHxzFCqr6Gy+kO6RIfIUiWTXAFbHJYDMXKtESY4JNRQhEj5Ee/uzmHSR/PEgppPrY496vByvnseEZxWulINht8RLcyxME/bhVoIcsjegTyicdqiR2R1jGHQ3IWcFEWc5ujBpi7blGPEUGIgFWhhu2w7KGHbnVA7XLeNQX4CfbvkJiZxctF/eTQV+XvjJ90jFJyfYaMBIzhT2VS6ltaEC3Wagsm2I//rzPUzpPs7OKcUkLRJXVR5ic2AmrfYGyu0TpAUzfRSxt2YGh4qrWaeO0Df+Q2JqjF+d/issEQuHtrXiP2xGETL4c48w4hqkOQ9qzHtJKTFGwwaCok5GfmN7Ngl2io0GyoJjLD2g07Bfx5AQwW1kdOFMYoXl2KoLGIn2MjDSxshYD1IihTWlY04bsaXtlAcz5I5EsKR0FNmAJoKsqsjqWw6RAaCKBg6vuIrBhnzijlEWRLayzr+TkFlgS62LnpiBjcliLgl9kF9Oa2DcbMBwdIKc4QB3qr/iyZzlPLvwIkyZBN5QF0N2NyW7R5g93MiUaCeaLNBdUsJWVhATHNRLo+QZ+limZpiIrsSeEOn3JViZjuJIhOlJHeaQC04U1NJbXMVwbhG6KGHIZGjobGHeiUbmtjRR29OFwZyDZcGNKN4K/GYVh3mCiJJhNJYiIyqUSwaK0ocx6uuJKVHG00V0pkoZTBYSkHTSUgR7LHRyUbK/QDYQlhx455/OLbe8SxtcCIJwDvBTJj1y7tV1/duCINwEoOv6XYIgFAH3A4VMDnF/T9f1P/ytOLOC/n+LWw90sOjB9fxs9m+YkUyxqeRWVhz0M69xLyt7uhn4vIOWwxcTIUVNxMLYeBiD9QycDhOr31eDwyAy8WQLwZoD/GLMzyv1ZxKx55ATD1HT082U3n5qhxyo5npkJUlV13PkB4/QU5bLy9MWkxv0c9qR3fj8QTQRkjWFyHPOo7kyn8fGX6LeOp3l0lKGTgyRjBkQxJObSKMzIukETBrndayncPQYDY/dy3gyzgs//yER/xhTFiwhMuFnuL0FXdcpmz6TgiWruFPKZzcqQjTDnC0H+Oi+h+gucOFQU1xaepTnQ/WMp73kO+1U5LSxy7yE56acRnNRJXO6djGh348iKBRGKojLCSLmCZJy7A31KqkiXkEj16oQS5fQ6FgJmo/LkiqfSm/FvnM7wVYDsWETuiCQmqFhOOd8mrol+o83IUoSuqZTt3QFs889jXDjNkaPuTksjrDJ20mXtQdRm5ytKWpQOawztV9H1HSaS0XS1hzOSS9m9dh0TAEFOUdAqk3jD76G9vxhiFvZPe+bZJjc9FqWdbxSBz6pjUzRAPc4xolI8LnR2/jW1FyO5shUDAwz0pjmDvEJcsUxvjrv02iCyFnbn6Oirx1VkDjqbKDVXsW4sYD6SCvTnWHcioyQmYEzYSVkHMadbmXcAcdynfQUVzFYUIYqyYiaSsloN6VjzeRMNCLFW4hZMgRtAiEbRCwgiBI6BkQEZI3JZZt1EUkXWJqMcmNolOpMkjaDhSfFhaTHL0WgnOcXmOjJt2BN9FI+ej/eaDf2tIAlY8aUNmNMmjAmDVTMmc/1H/zy22pL2YlFWd5VEqrGB+7dy/n9P+AnVW1MTZrYVv0Lbr3/R5T4J6j1jbGp5hIMBitLq6dzaGM3knEa5TO8rL6uAbU9QO/6Z3l4Rg8PC2cSNbkQdA1dEBE0lXOaE8xsSSErAmVGgdrMCMnoLuKDHVhDAxijSXRJJ1lrZKJwHv3ihcQU5/833yqTC0JJogCqBicfsEVRR80EEaUElbNrKK4rRhAEEpEoQ20tDLW3k4zFMBjMjM+cxiNTiwnJUHsiwkUvP44md1M5GqTE5OWEbzFBgxMEAZOkIcoGunIqSRgtWLQ+dhU9QIYMeaZ8QqKRbkclBs3B8v2HqRgYx5wWGTYX0LW6np1FKynVurhz55eZeiRCsMuBEgfZl4P7iiuxXbSW1177Bsc3jCMgs+ram6iZv4w9Tz/MkZc2oCoaXkcFySl2BuboNCfb8Q9A2vZ++r0RHJET5CSOEWeQjDjZO/fGZDxJD3a9kNXeKVRbmkn4DiAgYevPw/G9MQwXzifnE3fR0trD/qZGIn1RvJF8NH1yz1BBUPEUODDFFXb4BLaWG5lWaMe3vZGSpmcpSvfQK7rRBIGuslpW7TvMb2vfhyIZuKn3ZSYqCxg2z8Ez5mfQFaE330RfYRl9hRVkjJM3ktpwB4tMGnW5dVTpCtFMBn9Kwd6vUdGZQFXCtNtG2Fw8SJMtTkA3YkknafCP4xZMKIpCaXqIy5Mt1CshBkUrG/Tl+AOXYUr72D5NZEe9A0FXaRjdToX/CKCiCToaGrqgoZNGJYXNGGeuWsANNz36ttpTVtCzvKu8MBZk20+PYsy9kfUuiaj5DMzCWj74598yq3eE0AovTdYlXLruEl75fTOCmMvcNSXMv6CGLfsO8duJVvaZq8gIk43TlEoyPR5mWWeS0m4HYUUnRxJoKLMRdMBQ7CWsJbuw5Z8ATcCjz6ew5v3kV5zP5LQKuP/o/Tzz3L0s7K/AFBSRbD6GnFNp1AswIzPTbWNWpQeXLBNt6STS3I5SUEFUlVFVkUlL5P9/L0uAhKyxcZ6dYxVmCgIZrnjuMYzxE8zvHMKdluguW8OwtwFVFlHRAA2/20fCbMaZjOJ3jPPsvApkJcPa156ltL8DBAhLTl7MX02wvpRYtYeL9m3kw/sewtasgw5jpSX01tczVlqKrqrQ0wITY+h2O+mSfASLFdEsERWH6FODyCN+anqtyKpAf3Ga2Lxi9vm9yJlyrl12Nr8OJZkw6CzrSnCJJ0ho4kUO9+7kRFGagHmyLoyCTjku5gTnUR+pobDlEXL2Bei/bhq7xQpGjQFKyyo5ffoqKo69THj3bkaVKsYzMxhXq0gpk6YvTUuQzOxFih1CRyM3JdBRW8Hvz/4QxnSShUf3sXC0lQGxkBG3nd7CfHqLq0iaJ28SvmgMczrKByae45qRZ/lt/uU8WHQhcaMZVZQ5bzDDhzvTlCR0TjhE7q42cCg3gVOPIMdMeKNJFnQ2g5ognzHOYgdT6CIh57BFv5TGobOwayYCdXaen2OhW1c5L9fFN6cUU2AwoMUyqKEUscEJBk/sIRjdTdrahp7bg/lAAtE1m8W3/ulttaesoGd5V/nolmMsfKKVx+d+iSJFYXP51zhtbyvzmvZz5vEuNlx8PvmOWcQ63ERMMoZLSmnKkdjiD5NksoHLWobarjYu7Qkx11CCf9xOT1rHJMKU+hwStg5i6os4SvciGRMY0lZKaq6jqOQKzOaiU3lR0ml+9/A3GNm6D2fcgO7KY49jFgfkSgoEmUutNq6+ehZ5lW4AEo1N9HzgAwh1tWxxG0knk6y89kZmrDqbeDBFf2uQAxt6CA7HWHBeJTNXliDKIqIoIIoCyViE49te4diWl9hhsLPptAvRRIEPPvFj3JEka/o6MPh1IhYr9kScoMPGRKWJdoObx86+hq6yWvLGBjl9z0tU9LefWkb2oGMWg7nV+EoyVHQPcMlrm3CHgmQcVkKLUmROS9AvL8UlXUp8aICR3VtRU0m8M+YhV1fRNtZMf3IEdyoHR8ZBRkpjLDdRP7UGS/M4nVu2kUkl6XVO4dqPX8/c2Q1EFJXPH+rmiWiEvEiCm0ZfYVrlSyj0EQ5KDB4S6dSdHC0SGbXF0QUdS0rnx3erRM3w9WuNODQPLsWBS7XjUqy4UgJe0nj1BJaxNPqojYBmxK8GQc+gW+oxGxYjSjkAhFxj/HFxHn6XC1s8Rsw2aRpzJBIsGVOwY+S1XAMRU4b7Gr/EqsBevlP3aV6ruIQCSeT0foXFTVEcMZWALcVxw36iZdspLmhlYrSU1palCLpMjmajygoLTVvxhbaB2cVg/nWsb1xKKiYRyTXQsTaf59Jxik0Gvltbwtm+ydUqR/uaaTv6OBMT2xFtA5jdk/Mr9SEL4uO5aJEaktOtnPfzn72t9pQV9CzvGlFF5SO/2cN5Aw/xg/rtzEy6eHnKL7j13m9RGAgwTfezfu6HaMpfTEuxzEjOSVurnkERDNjUGGvbh/hQa5oCuZiejM7xpIqqC+R5dAyVOzH4XsLkGgRVxjuqUu5dgXv17xAkw6l8pOJxjmx6kW3PPAKxFBO2HPbYF9FhqWCZw8qFUThjegHe99UimCSUVIroYD8dN91ELJngUJ4Ls89H7eJliKJEMhohEQmTjEZJxVMkk/NR1RJkuQez5TCCoIEgICCAAIIgkEkmGcuoPDP/TCZcPq750y8RBJ1Z40MU90cRNZ0ej5tRp5mMLKPYREZtPuzREJKqYUkrRC1GRtxFXODwMzFgoOrIMSRNw7p0KTlXXIFj1Uqiaoind15EgTJMx94aYsdkcnxF5M9ZxHNsY6d8EBGB08PzWSsup2jWDI71nKC9vR1JkpgydRqPtGi4BpuYG21ETSWZsmgpCy+6DJNDYk/3H0hEnsFJkNFkBZnmM7GmSzEPvMiUgb0U+aIIxSnazAY6VQeBJjfLDiXYebbOjgYDyQkbYatA2KISMaRQRA1JhbpeBzM6XFjSEr15cY5NiTHisSILDqaoToyqHTUsYEg56Xc7SJnKyaGSjw9AgdVEz2kFmEsd5IsK0577COburURXfpse92kkD4/jbgdzRmZUCHFI7mLQMMT0hldx5oygptZgaV2Ha1gmr8hMbv6fkU78EV00MOj7AC+3nkUkbmbYDK/VGRmf7iKoqtxYksvtJS6G256jv+tFEkojRlcYQQBNkZDVKWSUMxl6RUcbyyVuKwHAPV/lqutXv602lRX0LO8aj/WNcOLnx7Hnf5L7cwHLxSAs4donfsX0vlESdQU8PeujbJhVTIUSI22IMqjn41ISXNEZ4UPdZsy6QG8yyEFFQ1AcWM0RnNOex1W1FUFUkfWpVPgjFLccRV71dVh6y6k1SzLJJLuefJRDG59HSSUZM7roM5eD2c5Mt0zJeBgpFUdz6aTFFMlYlGQkjKoob1kmg8mM2eHAbHdgsTswWqzoOkQCxYQnyjCYIngLTiBKqVNulDC5MBm6jqKqbHXk0mK0s+7VPzPuyWVepIXCniSesQQJo5E2Xw5xs4guiOSFY/TZfSiWDFVRnaKRUcypJAG7k+NVVVhyjOg+L0azBaPJikv0okVVmntfIhVT8UyN80qxyCHTCE7NzjniGVxQuJaCilIEs0ji5M1pdHiE5u4eBiaCCIKASdfISUXJjAyQTsRAB2d5hIL5Y6QTRjYq69hYcSHnDu7kxvYtTGMPRiFNSJOJ9tsRR3XyZoXRNYGWTYUQ1xn7SppAxkfj8RWoBis6IIWHsAf60RMSNkcaU7FAS34VPUYz/cYYo8YYghZG1CKIWgThL1YUqUgXcUHlBaydewFaRGN0oJuKXZ/HF2nmBWEtscyZzFIqsGNmwhhj1NmGOfosPnuEiXmQJEC5ehumV6YjGERyFo5jOf5ZiI0y5LqYl7vOJZx0UTnLx6tujT/ISfAaWGPt5wPqXoTgDnRjP6Kko2ugRHKxGhZjks6jt9vMSFMEQTEh6CpGpY/c00pYvnopnkIbwt9YuvhvkRX0LO8aVz+5iyWbouyY83HSCOyo/Bmn736N+Y1HOLO5h81nn80TC69gwCOTMUg4lAwf6lR4f6+C0STQKfdxojOOJlciGWPkzngSV9U2UF3kOM+nrmw19qc/A+NtcNGvYeblAKiqxuYn/kTTc4+hZ958SSFRkDDJVixeN1aPC7PNgeWkUIdf24bxWCOjPjc111xHzYLFmB1OzHYHssHwpvEBdB4aY9P9zZjMEutunkl+xVsPvvYkUnz/+1+n+vhR9i9Yzo8Tv0Xr0/EfKcIQCjDiySMqGAi5rBT4BykIxxF1nZaqev50+pksNTSwcCCComcwSSJW0YhFt9AS3kdjYDuqQaBr/jDrpo+j6wI7DnvwHPQha+Jb5gkgJZuQ84tJO9yYHROUlTbjdPQzfszLeKMXNQ2GAolSXwtn6WM4NAW/wcUe63IiPUtIWecwb10lVfkJjKYo0gs3k+lpofOlEvTqPAZu7sIg1BE8soTuI0fJJBJk7HbGV/g4XjCbTqGGgNENgElRqA4r9NolYgaZ81s7OH1gHNmg0erpYJv9GEPCEOhQmMjjA9ERroj302m+Dmv8QgwpEQrNuKeHsB29HSHUQ3jueRzJaUXJZPA1fhqlp4x0oYUC0+OU+X9PhEI2Bm5nLFPNlAV5NJxVyt2jB2kP7GCmcJQZeiMGcXJ/08SEBSlTjcN2DgaWM9CaZKgjCLqApkcpGG3GHW+h7PrV1L7vyrct4n9JVtCzvCsEMgqf+vluzh7YzPdnPsHMdC6bp/yET9z3DYomQsz2j/PU6hv4/RnL0AWB6ztSXNIdxWjWcJ5fwYMvPk1O70wQBLxT1+Op3YgYmU71rA9RXrMGcbwd/nApJMOk3/cAR4xz2Nfpp3PbS3hObMGkpdCBXlspJ3KNzKlz8KnlHyKzcwKtOYa1Nhfv+6ci2V/fXi2TTLLza18k9+kXCZSXMP2hP+DKy/+Hyj3eH+XF3xwlHkqz8uqp1C0qeMuwqVSSH95+I3o0wRMXX8d3e3/FjL4j7DtYxpTRCPJJH3vVaMRwyaV8tn4Rh3Pz+VGPzpKuBHp6cpVEHZ1YJsRW/5NEE+N0F8bZ2eBnnnUOZ9mmIZsewmGMo1suoN59HUoqRTqZBHQsDieC2cZnnu/g0Gia31y9gBm5TfT03E0otA9NMzHQX4045KRIaUYcUWn2F5LSDBTlw/wLLubh0rO4azBAgShxSVMaz9EQdo8JX4kDu0OlIfBDpL2vMLTfTd8lS2kP9JKcMBHNzWfz/NW0ldWBIFDIIFP1UZa372FVaDfuWIQ/s5Z+sZBXG5bQ6fEwr7OJ+X1tCLqOmE4SxU+/s4fh3CHGDAI2VWZFeBHLk7Moc1vwRF9E8A8zITcw4FvCeChAJpJPJpKLhoBNHGe1+ycUG5vp4ixafJ/BUSJQ2NBFV/A1QuG9uITJJSmSUSPJYTd2ywKcjgtIxYvpaw4QHp8U+JB9hCCHWXmkmfK+DuwXn0/x576A5HK96f//dsgKepZ3hV8fbCL84Ch5RXfwk/w0kv0q0kznI4/fRf3gGJo3jydPv5n1s4uoC2X45t4equtreNW6i9B2H3rCi73oMJ4pr2Afn06N+1xKPrAMQRIIn3gVy5NXk8LAVx1fZ+OgnbrgMeYFD2PS0+hAv8fCtmnj6K4wVzd8kI+XfpSJPx5HGY3jWFWG88wyBPH1HtNodyebv/sNGrbvQ/T5qH/hBWS7/W2VPRFNs+G3jQy2BZlzdhmLL6pGFN+8dzYx2M+9//VxBj0+HrnwY1w++hK3tTzHluYCfCPDyL4iLGfdxicqLSiCwE8PxpmNAWO5A2OZE6nUyrM776f3+ZdR0dgzLYTNMoNzQ6tYqBRhVnU0KcHQ9HuI5h/AOr6ImvQdGH05GApsUObghqeOsK9rmJ9fOIpLe4J4vANdymE07kJu62BVMIJPU0hhoIVqhr1LyRgq6Nq7k1QsRtXcBbjWXsyXEwY6EikuMNk460iMlD9Fu6DQYdaZE/sTOYcaCRgtKHYj5Yt60cotvDz8RYpiXkqiSbTwINFkDFXIUCT1cZr4Kk45SGvxlRi7lvLLGXZeLDJwjkXk02IcEjESo8MUNv6BZCrFVuMKXnH30OQ+SkbK4Ex6qR1bSO3YApwpL4KUwmQJ4kp7yGQm8Jg3c7rzeWRBpclxHoHC2QjebSiGwwCEcdKaqSXTKlChzaGy6EyCYwb6TwRQ0hqiQSCZ7+eA6VUGzIf56CGZWTtHMJSWUPiNb2BbsuRtXT9/i6ygZ3lXuPK+LSzfA60zb+SYycjx8ntYsevPLGo8zsrmbnYvW8HvVn+YYY+RL3Ucos7YQV9nHonh6Rhso9gK9lFVMIuixjImpuTQNjOHA71BrO3P8bnEj+nXc7k1/jHqMiMUjB5D1CZ9owc9CQ7NT7KwdgWrSlexrHgZwok4gSfaECQBz/unYq7NOZVPXdc5vPF5tj14D4tb+3FoUP3npzCWl/9T5VdVje2PtdH42gDl072s/kgDJov8pmGbt73C+l/eyYFphbyy4maKUqPcObgBV+MCOgtL+excJw5B4IGcXKZVe5FdJmKZGE8deZTGh58ib0DAn6tRfeV5XDD7So70pHnu6CCbmkYwZjRmWkysLrRgd9yFI28HhngJZYc+hSHuQ5WjdJe+jFq5BV0OkVBsxAfDLB0IU5JRUUSJYNki3PM+QrLkdA4cbWbfvn1Eo1E8LieFosbIkf2kYlHKZs+nZfFqfo0NpywRVzV8Qz2s2LuJssEusFiY0dbNVNs4gQ+ew2jhq6hpCx3br0eJFCHpJgT1zV1CRUFBFkR21NvYMN2CoE/uCPTfL1FnstcOiGocObUPObkTId0yudepsRrRuhCDvAKjJHBL9++4cvgZTlir+EL5LVTYGjnL8iIiGi+p69gprmDugJ30wST1Bjfm6OTTkN1jwliZ5oBpK5uVZ5GNIh8Nzmb5o80wEcJz7bXkfvITiBbLP3X9vBVZQc/yb6c3EODbdzWycnQ/P5l5D2VqAduq7+TmB75GsT/Kou5hnrrgo/zmzOXMTpzg2vZ9TLSsAXRMvk7CnbsQG1ZxIFREo6QxcXKa+c3mTXxav58Dyix2RKvQh8fQmWz+GatM4cIlzK1YTqmpGCGloyUV1HCa5PEJjKUOPFfVI7tNp/KZiITZeNfP6di/m6UJHXdrJyW//hWOVavesbpo3NrPtsfacOVZOOfmmbjzrW8abuNvf07jlpdYv0RkfPqtDEteLkq28qKllgqLkUdmVlNsMdEf6efhEw+za9vzzDtkw6RIFK1bzuUfuB2DbHxDnIm0yistozx3ZJAtJ0ZJKRmW1j/IB4oPIQgGJiILyLftQ5IyOPw5VPRFyQ0PARIx2zzkqZdhWHopktf7hngVRaG5uZk9e/YwMDCASZYoMQiEWhpJxaJ4GmZzrH4e+cf2Irccw+xys+jC9+GonkrgVz/DtelVSk/zc6K2lIlZKSTJQE31LygrW4aSSbF313VEAgGqCr+FvvURYmNjxLUcInoe/vR0mvJz6PNJmIVRjFIA2VeJsbgIQ+Ao4uhRNKMFtWwpqrOYgf4NjCb2EYr0kNb8SIKBFUmNKwNDRIouZnvdcuYk7sal9tNtWMKBxPXkN5kpHEiTiSpo6ORVuCif4aLNdZjHRx+kO9JNniWPqwvO57THW0ltfgVTXR2F3/oWlhnT37Fr583ICnqWfztfe34zzg0i1eVf4kt5EQyOq4gzhRseu4+6ET8m2c59F36aLdML+F7zo8SOnU3SEqVDnqC69wkaHfW84juDIkFkls3MLFFider3BCb2cTBQSSQjIAtGFD2NQTAx03MGVY6ZiMJfDPhJAqJZRjBLWKZ5ca2pQJBf/72/uZEXfvkj4sEgK6fMwPTYE3hvvJG82297x+tjoDXAht82ous6Z1/fQNk071+FyaSS/PGLn2Z8fIDHlw4yrfh6NlgXsiB0jAcbP49biZIRRcKawK6RSlonCvBa05w7PUNujglkE8iWyXeDBWTz5Msw+Z4SjLSMpTk0lKQrs5/Z9cexGXTs4xlm9MawxjUSeXMR3ecTjy8k2SucstHL+VbM1W5MVS6MlS4k2+sDw319fezZs4empiZ0RaFI1kl1t5GOxzDZbJQuWEbC5aWzu4d0Oo1RFFm3aROWdJQpZ3WTrKrlUK2AqieZNfMe3O75JJOD7N13IQaDhwWzHkZ+5hY48Ty62Y2eDHNcuYRS+Rh22nlt9g/JsZuZduhbGGKD6PM+jHDWVwl1jXCs82OkzH2Uxj9G1cqP0Xz0Fzx7+Lest5oIiwI5BiNzzDFO8xawYurX6Nlfzv713ciySCbfxHNjQa64yMe45VWeanuKSCbCTN9Mrpr6ARYdijP+wzvREwl8H/sY3o9ch/A3BszfKbKCnuXfiq5pvP9XL3Fak0yg4UaetlvpL72HpXsfYtmRHk4/3kPTrLl87/JPItvGuWNHI9GReu42hnn/0BMgOykqvJKZRgsut8BAvB3/4CN0B0UUXUKyWNEzCpqSZtrs01l81uVYPC5Es4xokhDM0qSQy2/uzaFpKruffIzdTz6Ku6CANedeSuSOz2KZO4eye+5BOLkH5DtNeDzBi785ysRgjKWX1jDrzNK/8nrwD/Txx8/fRtJn4A8zGvlk4fnMy4Q4MbSHcHwEU8xJsr2CeFxgfp2NZVMNyHoKMglQUqCcfM8kQfmLl/rXnj6aMLmbvZY3C/Osq2DaheB4fQBYVzXSA1FSnSFSHUHS3WH0jAYCGApsmKpcmKrcmCqdiFYDoVCI/fv3s3//fhKxKE5UIkjoooTdbqeuro66ujoqKyvJHGuk56qryDn/dAq860nKKofml5DUw8yY8St83jOYCOzi8OFr8PnOYkb9zxBeuA0OPYTunYLgbyODzFcy17FSPMDZ0gGOa6V8MfMRRqRp3OEcxTP9p6hykvHoZyF/Aae1fZeC7mdIlS2ma9Fi1nf/ib0xieaEgC9SypqeD2OLeCif46bm7Eou+uPD5JXuJSQcRkRkdflqrpp2FfVJD0Nf+QrxXbuxzJ9H4Te+iamq8l9yzbwZWUHP8m9lz8EDPPJcnBXRffx26t2IQiGHK3/M9Q99hbLxBCta+3n2/Bv4ybqVfDLyADkb1xKQFUpMLxEeHuKsvKsJFGb4hXgXpe0a+X4LkqCRV2wkKRQQ6OunsKaOVdfdREH1lH8obxH/OC/+4kf0H29k2oqVnHHZVQxcdTW6olD51JPIHs+/qFYmSScVXr7/OJ2Hx5i6pIAzPjAVyfDGG0/za1tY/6sfk1yQz6O5kytQVzuqOG90BuGtR7F7vKz72G2UNsz8+xPW1JOCnzwp/ieF3uIBZ+HfFYWunBT4juCkyHeHQTkp8IW2SXGvdiGWWGlqO87x48cpLCykrq6OwsJCRPGN5Rz+xjcIPPIoFff8FEvTd0mPHeLw4hqiYoSGaXeSn38evb330tb+baqr7qCi/CbY/FXY8TMoXQyFM9APPQy6xuCcT3Gs+ErExjBFo1sZnf5r0qqNu098gsxYmJ/Jv6BUGOWBnNNwTBnEawnQFFhEc/BqSkYsuHtVkqYYr1Y8zIC3BVH1kBZHcRpdXFF3OVfUXUGeycvEgw8x9vOfI0gSeZ+5A/fllyOIf9sN9J3mbwn6m4/QZMnyT/DHQ61MGc6nqOY5hmQZ0bEKV+AwOTGN3HicCU8Or9bNxEKCqf4xhjUjRk4w3tvJIt+57MvsoLvpGPPjJmxGhTl5PRgrFvDKvnHMjghrbrqVhtPP/IcbUvv+PWz8zU9RMxnWffx26pefQd/NN5MZGaHiDw/9y8UcwGiWWXvjdPa90MW+F7oJDMdZd9MMbK7X7frTTltFb9NRmra+zG1XXUNZaR0jj7/CcMdhpq1YyarrbsJktf1jCYsSGK1gtJKMZehrnaD7WJJoYAiHN4DDa8bpteD0mnH4zNjdJkTpjfUryCKmciemciesOinwfZFTAh/dPUh0+wAIUFxsp6pyCbLNghQ1oY4mwG1CNL8uObm3307k5S0Mff9XVD72HMaXv8LcnfdyZF4JjU2fIqOEKS39MOHIMTo678ThmIZ39Tcmb0Kbvwp9uxGmrIFzfkhuwsvcP7czwp8ZmfUwdksdy+bew1r3U+ibvk7E42P39KVUaE2kqORo/DP0D1ZQ3RrDmtE4bFTZapZQgqsx6F5E8yAXVL+fr595NWbZTLKlhe4vfoJkYyP2lSsp+OpXMBS8tTvqu0VW0LO8o4THRhkP5tCgpzgmdQJOxpzLWbD/PgCKxiL0V9fRUuLhdP1FYkMz0QUFceRl3MY89k1sRNMUplRXM1fcTZWxlw0jM2nbN87sNeey9LKrMP+DroRKOs1rf7yPQxueI6+ymvNu/S9yCosZ+/WviW19jfyvfBnLrFn/gtp4cwRRYOH5VXiK7Lz8QDN/+u5+zrl5Bnnlr09COvO6mxhubyX29AGaMruQZQPnfepz1C1Z/g+np+s6geE4Pcf8dB8bn9wvU9Mx2w3k5FsZaAkQDaZO7esMIIoCdo/plNA7vGacPsupzzaXcVLgK12YKid9rPWMRqo3PNl77wwS3TkI6hstAIJJQnKbkFwmZLcJ54U3MfG7rzPy03vx3fBtpKIFzN5wG8fqbbS0fBklE6Z+6neIxVppbLqNhQuexrL8U+CbAoKIVrGa8Mu9RLYfYLz+T0wUr8fnXcX0yi8jPXkLSudLdM1poM8+hiQOUVvzVXzuyzA/1Y2xaQhXnoWVH5zKDVUuRsJJhkJJhkJrUTWdi+cUo6fTjP7ip/h//3skl4vin/wYx9q178gEoX8FWUHP8o6y8dUtlE8UUpG7jQctJpx4GBNs1HWOYkuqOFIZOkrnkDDqnJd6idHBL6Aoo4BOODNB/YozmDu3HN8rt5BKZXi0ox5DxRw++JmbyKuo+ptp67pOLDCBf6CPicF+Jgb6mBjoZ7yvh3goyNxzLmTFB65FNhiIbt/B+C9+ifP888m58sp/T+X8D2rm5eHKs/Dib47y1I8OsupDU6ldMNnrM5jMnH/b53j4S3dQXDeNNTffisPj+7vjVjMag21Buo+N031s/NTEF2+JnblryqiY4SOvwnnKN15VNKKBJOHxJBF/kvB4grA/ScSfoKfJTzz0Rhu8KAs4POaTPfrJnv0p4V9UiPOsMtBAjaZRg6nJV2jyXTl5nBiIosWKkQvnEHjoHtIDxYj2EozWnzCt8Zu0TonQwQ+Jd/ZT4/oWjdp1HDl8E/Pn/Ql56rkkmv0Ef3KQTCTM6On3EzLsoqTkGmoNp8PdqxmyRWhfXkZaH6Go4DKqq++gr0nl0Z8fIBHNMHdNGQvOrUQ2To6ZlHqslHpe90CK79/P0Je/QrqrC9dFF5H32f9CzsnhvUxW0LO8YyiZDC92h5jvz6Om9mWOmYykHctwBneSFxLxJBMkzWaeXriEORxEHs9FV43o8V0YRBMf/vFvMQ5uRn7ueoIpA+tDS5l3/SepX37GG3pEmqoSHBliYqB/UrxPCXg/6UT8VDijxYKnuJSKWXOpX3Y6FbPnAZAZGGDwjjsw1dRQ+PWvvau9rdxSB5d9bgEbfneMTb9vxt8fY9GFVYiigLekjJt/9wckg+HvymM8nKancZzuY376mifIpFQkg0jJ1BzmnF1O+XQvDo/5Tc+VZBFXrhVX7pu7VCpplchE8qTITwr+f7+PHx4jEXnjzjyyQcSZa6Fiho/aRfl4Z+W+abx6RiPVUU7PVZeiDD2F9+bvoIULCE/cR3X7N5CVQwwUPUK8qYf8kY8wMPenHHziRoq7PokeyaAXpxla8TOiqRPUVn+B0o5hIgcuoWWql5DNgtNRw6zaryHpdbx8Xyudh8bwldo57xOzyC1zvGmetFiMkR/9iOAjj2IoLqb0nnuwL1/2/63/9wJZQc/yjtG2dye6XoVZDHJUa0UXPERtC5l7+A8ICBRNRBgtKKG7wM0X9OcJdq9A1zNoSh+zp5xN6KVvUtx5P0NJB11T7+DCi64mOjHB8e2vnupt+wf6CA4PoamvL55lz/HgKS5h2mkr8RSX4ikqwVtcii3H81dCqKXT9H/qNnRFoeQXP0e0vrmA/TuxOo1c+Kk5vPZYKwc39uAfjHL2dQ0YLTKy0fiW5+m6znhf9GQv3M9o9+Se7PYcE7WLCqiY4aW4LgeD8Z/32pGNEjkFNnIK3tx2n04qRCaSRMYnRT/sTzAxEOXQpl4ObuzBV2qndmEBtQvysf3FPADBIGKeWkbeZ+5g+GtfQw8dIOfSS04W8Hnqdv4Sue279JTtRC5XKBY+wkDBPUTs08lxL6NN+hqZTIDZld/E+crvOSE3MTDXjcHopL76sxQUXMKJXSPsfHIPSlpj8UVVzF5dhiS9+fhLqqOD/ltuJd3Zieeaa8i99Zb3xDXy95IV9CzvGBu2b6dqbBlTC/fwJ6sZg2BDFz1U9QYxZWQ8kTivLZ5DsdBDnXqC1qGPoaZbMQgS7vFfUSIP0hP1cnx4NqGerRx8bgNpw+QlKogi7vxCPMWl1MxfNCncxSV4ikr+oQHCkW9/h+SxY5T88hcYKyr+RTXxjyPJImd8oA5fsZ1tj7fxxPf3c87HZuLOe6OYZFIq/Scm6D7mp+fYOLFQGgTIr3Cy6IIqKmZ68Rbb/+1PHUazjLfIjrfojeMb8XCa9gMjtO4dYeeT7ex8qp2SuhxqFxZQPScX48mZs+7LLyP0/HOM/OAH2E8/Ddnnm1x+eNknqSmZh2HrNbSX7MUjT+DzncWgcB/D+qNIupWFjpsIbPkcjSUiqsFGScnVVFV+injQwHM/P0r/iQBFU9ys/ODUt5zUBRBev57BL34J0WKh7L77sC1e9C+ts38FWbfFLO8IYz1dfPqhl1jUP4X313yetd4YQccqRD2P65/cRWEkzeyuQT701V9ybu7DzOkPMrTzZtKRp5gmNrKm+hBdgzkEd9sxn5zMAoDTiaGmGtv06Vim1mOqq8VUU4NoMr11Zt6C4J+fZujzn8d7w/XkffrT72Dp31n6T0yw4e5G0GHNDdNx5VlODmj6GWgJoCoaBrNE2TQPFTN8lDV4sTrfuif/XiE4Eqd17zAte0cIjyWQDCKVs3zULSygtMGD0tNN14UX4Vi9muIf3/nGkyMjDK6/hOPeIZxaDprNiyDIVPtddKg7iThk3LaZ1DV8D6u1lqNb+tjzTCeCJLD0khoalhe9Yd2ev0RPpxn50Y8IPPgQljlzKP7pTzDk/2MLsv07ybotZvmXc+ilFzBL03AZ+ziudpMU80hZ5zLz2KOIukx+KM5YXiHRXJFl+jZ6Wz6FrqXQ1QHq8zuJpo14v7WbCl8eajBIqrWVVGsryZYWUi2thB57nGAqNZmYKGKsqMBUV4u5rg5TbR2m2loMxUVv2TNNHj/O8Ne+hnXRInJvvfXfWDP/OCVTPVz2uQW8+JujPPvzw6e8T1y5FqafVkz5TC9FNW6kt5g49V7FnW9l4flVLDivkpGuMK17hmk7MEr7/lHMNgM18/PIu/pThO79Ia4LL8B++umvn+zIp+jSrchbbqBR2Io1FMOelDjsSmLSHUyf+i3yCi9kYjDG+l8eYLQ7TMUML6d/oA57zpuPGwBkRkYY+NRtJA4dwnPNh8i7445/y2zPfxVZQc/yT5OKx3ilqZUavYGGol28aLUABkySl8rBFJImUjw+wTMrz2MVG5E1jYy/HC3TwRTRSKkjRKe0iOq8SQ8P2eNBXrwY2+LFp9LQVZV0by+pllZSrS0kW1pJNjYRWb/hVBjRbsdUW/sXQl+LqbYWVJX+W25Fcrsp/vGdCPJ7/7J35Vq49L/mcXBjD2abgYoZvr9pLvjfhCAIFFS5KKhysezyKfQ1T9C6Z5jjO4dozFRgWf5ten75CovKG/BU/IVnjySTt/o+Zh3+EcfGfk3cAeWOtVTM+SGCbmHv810cXN+DySZz9kcaqJmf9zdNT7Hdexj49KfREgmKf3wnznPO+TeU/l/Le//KzvKep/m1LQx551IyoDKNV7jN5iBlmYE1tI2SUSsOZXLxrOeWL+d2vsFQ60rAhKr0UGlrBqDowz/5m2kIkoSpshJTZSWsXXPqezUaI9XWelLoW0m2thB+/gWCj7y+o7pot6OlUpQ/+ACy96/XUHmvYjTLLL6w+t3Oxr8USRKpmOGjYoaPdEKh8/AYxze30dm/nM7vHSWvwkntwnymzM8/ZVbyzr6DheOrECQTlpwGhjtDbHlwL4HhOLWL8ll+2RQs9r89mOy/5x7GfvJTjBUVlD/4AKbq/4x6zgp6ln8KXdc5+NKLODxXUGA7Srfqxy8VophnUXL8GWTNgS+WIODKobCwmxyCjJxoQNfTWPVRynKHGU7lUlja8LbSl+w2rHPmYJ0z5w15UoaHT5lrUm1t2E8/7Q1hsrz3MFpkpi4pZOqSQjq/8l3adg0Q8F3O9sfb2PFEO6X1HuoW5VM5Kxerby7ppMJrj7Vy7NV+7G4T531iFuXT//YNW41EGPz854lufhnHurUUfvNbSPZ/cNbte5isoGf5p+hvPsb/Y+++w9yozoYP/2bUu7TS9l7tdVm3dbfBGAM22KaG3kIPEFIgCYRUSCMJoQSSAAFCL6GZbgM27n1dt/fetNKq15n5/lheAqGDwYFP93XJWq3OzJyZkZ89eubMObVJDSX9MlWFW3hDO9a31yVYKBwQUASF4p4B1sw5kmW8jD/gQomVIkvDTBUHsWrjBMadc0jrJAgCmuxsNNnZWBYtOqTrTvlqFP7oaqQTlqM60IPtr/+iqWaEph0DvPHACGqdiuIqFwOtPgLeKJOPzGPOSSVo9R8fzqKNjfRccw2J3j4yf3oDjvPO+5+94/Pz+npdVUn5n7N3zasEs+ZhlGKUxjfwssVFQluCK7GJ/CEjerRoZJnmGTkU04Z7xwQE0YAktZFtayCSVJN18o8O926k/I9RWSxk/eLnxBobUV7/N3NPKuX838zj5GunUzErk67aEdRakVOunc4RZ1Z8YjD3rVpFxxlnokSiFD78EGnnn/+NC+aQaqGnfAFBzwhNu7ZjK5tLvn0DHjlCtzqKqJ2Aue1ttEknzkiCmEZLSVktsYSOSP9kRI1MUbyZXMcoXcJkivWfb5q3lG82y5IlWI45Bvfdd2M97li0hYXklNvJKbez6KxxH9kN8b3keJzB3/2O0SefwjhrFrl/uXWsj/s3VKqFnvK5HVi7hu7McgqGJKakbeZVew4AFYKIo1+LLCqU9vazr3Ic09U78Ry0IarHo8ijVOrrUYkKxqXXHea9SPlflvmznyFoNPT/8le8956ZTxPME729dJ5zLqNPPoXz0ksoeOD+b3Qwh1RAT/mcpGSS/W++hpy7CJMwSk5kB89YMpBUTgrZTsGgEUWrwxIKE5msgAIj9ZMQRD1CvIms9B4GIjYy5qw83LuS8j9Mk5lBxnXXEt62Dd/zL3zq5YIbN9F+yqnE29vJu+uvZFx77deiu+oXlQroKZ9L6+7tjPp82EZMFLu2EEeiQ/RhVJcQ7/OgT6hIf2ecrMIJtfjazYjMQ1EUJie2YjdE8bkWfSPzmCmHlv300zHMmMHQLbeQHBn52LKKLDN89910X3YZ6sxMip/5N5YlS76imh5+qYCe8rnsW/MKnor5ZHkkppg28kzmdBQSVAtRpC4LkqiQN+ShPz8NrSNGf20ugiofRfZRbKsnLqnIPOuXh3s3Ur4GBFEk+6ZfI4fDDP7u9x9ZLun10n3FFbj/ehe2lSsoeurJ/6nxer4KqYCe8pmN9HTTdXA/smMmDnUXaZEGnjA5UQQtkzW15A8YiJq15AwNIVYGCQ3qiUpHIQgimlgd2a4ROsJ52PO+GTdzpHz5dKWlOC+/HP8rrxDcsOED70cO1tJx6mmEt24j61e/JPsPf0A0GA5DTQ+vVEBP+cz2vfkqss6AvUegNGMzCUFDJ4PYxBxahjUYY2pscR2ioiBODdHRVowpUYqiKMwUX0ejkpEmHZ5JJVK+vpyXXYq2tJT+X/0KORQC3pmN6emn6TzrLBQUCh97FMeZZ/5/m8pLBfSUzyQRjVL79lvEZ52KPZikSr2ex0pWgjzKXDlCuMOELChkeKLEzSIhp4p61QnIihVkP8XOVtxRI8WnXX24dyXla0bUasm+6dck+/oZvvNO5GiU/p/eyMAvfolx1iyKn30WQ9VnmDj7GygV0N9Dicdx33cf4Zo9h7sqn5kUDDJy/wNEGxq+1O3Ub36beCRMJFFAlv4ghvggj2vGRrObq+kiu9+Az6aioruD5KQETf0VjA8mEAQBs3yANEuY7vgE9JYPny0mJeXjGGfMwH7mGXgeeZT2U0/D9/zzuK68kvx77/mfnx7uq5AK6O+I9/TQcc65ND7wIC3XXYvk8x3uKn1qgbffpm35Cob+9CfaTzmVgZtuIun1HvLtKIrC3jWvYi4Zh60jxrj0LfiNWXTLvdhw8Fa4AEtEjaiyYIzGiFQqvG49H9VQNoqiMEfzHElZwHhkqnWe8vllXHstapeLpNtN/j3/IP2a7yKovvisTN8EqYAO+N94g/aTT6HXN8qbxx3LukmT6L/lj4e7Wp8o6fHQe92P6LniO6gsFgoeuB/H2Wfjfepp2pYuw/vEEyiS9Mkr+pT6mhoY7mgjMnE51liU8fJGHik/H1WinYWJMNE2AVlQyB6VUESFg+kl5ER8IKQhKmGKnV20BtIpXXbCIatTyv9/VBYLRc/8m9LXXn3/mOkpX8OA3r8PHj4Rdj0IweEvtCo5Hmfgt7+j97vXIJeWsuOYY9Dp9XjT0th18CDBTZsPUaUPLUVR8L30Mm0nLMe/ejWu715N8bPPYJo3j6yf3Ujxc8+hGz+egV/fRPuppxHeufOQbHffmlfQGoyM9hvIM+9ALYV5AgUBhYlikrx+EY8NZnTWES9TeCTnIsYfHEYQBFzUoNNIeDTzP3aezJSUT0OTkYE6Le1wV+N/zqcK6IIgLBUEoVEQhBZBEK7/iDKLBEHYKwhCrSAI6w9tNd8jNAyjXfDy9+HWCvjXcthxHwQGPtNq4t3ddJ59Dt5HHsF+3rnsWbGcYCTC+eefT1lpKQenVNH629++ezX9f0Wiv5+eK75D349+hKYgn5LnniX9qqsQ3hMk9eMqKPjXg+TefjuS30fneefTe+11JAY+2zF6r7BvlKZtm8g7Yin2jjCVaZtpT69mKNGCSdHytlyFLaRB1hiwjCQYKrMRixegD5SAojBX/xSjcT1ZR19+KA5DSkrKh/jEgC4Iggq4G1gGTADOEgRhwn+VsQN/A1YqijIR+Nahr+o7ypbAd2vgis2w8DoIDsKr18Gt4+GBZbDtH+Dr/dhV+NesGbstuLOT3L/eScfixTQ1N3PssceSk5PD8hUrELRatuXnMfiXj5944auiyDLeJ56gbfkKQjt2kHnD9RQ9/ji68vIPLS8IAtalx1H6yiu4rrySwBtv0LrseNz/uAf5/6Zy+7TbVhR2v7oKKZmk3zodm+KlOL6Tx8ovQBs9wMJoBKl9bMb5GaP9ADw2/VQmDmxBUDnRECU/rZcmXx6FR838YgciJSXlI32aFvosoEVRlDZFUeLAk8CJ/1XmbOA5RVG6ABRFGTq01fwPf9zPXXvvJpkxHhbfCFfvhCu3w6IbIDoKr/8EbpsA/zwGttw11pp/x7splmu+h7aoiOLnn8NfWckbb7zBuHHjyMrKYvXq1QwPD7N4yRL6c3I4sGED4ZqaL2t3PpVYezud55/PwK9vwjClipKXXiTtggs+1YUg0WAg/ZrvUvLqK5gXLGD49ttpW76CwNq1fJoJwiPBAC/f8Ud2vPBvKmbPZ6QhTIFtMwoKT8ejCEqcUsVE3kCMoEGixOch7hRYk72EcXuiCIJINtuRFYjZliNqv/njaaSkHC6fJqDnAt3ved3zzu/eqwJwCILwtiAIuwVBOP9QVfC/behazz377+HGTTciye9c8MsYD4t+Alduhat2wuKfQTICa26E2yfDvUcRf/4mOk8/Fe8jj5B2wfkUPfYoybQ0nnzySdRqNb29vfzrX/9i69atPP744xiNRrIzM6mprqbzF7/8zK3aQ0FJJnHfdx/tJ55ErKmZ7N/+lvz770ebl/eZ16XNyyPvr3dS8MD9CFotPVdeRfellxFra/vIZTr27+Hh666iZccWFpx5PhPPvgp7b5TJlg1sKTmVcKwOrSKwh4XYgypsqiTGDoXWygpyQgEsiVJAYY7tSdqDaZQedeoXOBopKSmf5NME9A+75eq/m3ZqYAZwAnAc8HNBECo+sCJBuEwQhF2CIOwaHv58FzQLOjVMb7Dzavur3LztZmRFfn+B9Ao44kdwxaax1MySX+FvitD+i8eItzaRu9xCYrKK1a88x1/+8heCwSDJZJK8vDxOOeUU5lz6HWwlZTz//POUVVQQ02nZZbfhvvtvn6u+n1e0ro72009n+Na/YF60iJKXX8J+6ikfeQecN5HkxqYedvs+PudvmjePkheeJ/OG64ns3UvbyhMZ/OOfkILBd8sk4jHW/usenv3tz9EaTZz9m1uZffLprNnci0vVRUa8mSfyT0IXqWFOJILQM5abL7G7ERICj0w9gYr2nQgqF1olQrpxkE5/GZlzxx+6A5SSkvIBn+b7bw+Q/57XeUDfh5RxK4oSAkKCIGwApgBN7y2kKMq9wL0A1dXVn/x9/0NMWnQMJ+3ajtTyBs/yLDqVjutnXf+hgU625DP0dhjvyz7E8jJ6Fk/l1aCEb1cYgVoUVEzKUHHCypNYryvghx0DHGzqx1FUxdk6Axs3bqSoqIhWoOGFF7AuPQ79hAkfrNQhJMdiuO/+GyP334/K4SD3jjuwHnfsxy7TF41z5r42msJRHusf4a7KQpZn2D+yvKDRkHbBBViXL2fottvwPPggvhdfJOPaa4lOnsBrd/8FT18P05etZMHZF6DR6gDw7HEzOW0jIZWJ1+JxTJKX8oSJ0X4/iDLZ/hCSVsuuiqlc/sxqBFEkT9hOKKlBm3kKqo+ZuDclJeWL+zQt9J1AuSAIxYIgaIEzgRf/q8wqYKEgCGpBEIzAbKD+0FZ1jKhSsfz7P2EZs5nUYePxhse5rea2D+SDY11dtJz2LbyPPkrH5Mk8NXU6mwM6Mkoms2judERBRYluBLO0hRNr6rjwYDtuTyff7boHfXSQ+zJKCE6bTUdHB3qtll2zZtH9s5+jJBJfxm4BEN61i/YTT2Lk3nuxnXgipa+8/InBvCkUZUVNM/2xOP+cWMQks4FLazu4t/uTL2OonU5yfvMbip5+Ck1ODv033ED76WegHRzi1Btv5qgLL3s3mNd2eLG5Y1RpN/DyxCsgcgBBgSF5LpYQmHLC6A+KNJVVkh72YZPHBt6am/YkdaNZjFt41Bc/QCkpKR/rE1voiqIkBUG4GlgNqIAHFEWpFQThinfe/4eiKPWCILwO7Adk4J+Kohz8MiocjybpaQhw0nW/JPbrHyP31vIgD2JQG7ii6goGBwdpffRRLA8/gqIo7Fi4gOjsKrJydfisPraHtmPcbaLbmcW9ZVNI6C7GFuvnpqZb+Hb/ajRIXNn9EhdPvJlH7dNYWpxJYfsgUbOJfQI4vjMP1zwnqHWg1o89q3Tvf63W/9fP//esBWsu5M2E93yjkIJBhv/yF7yPP4EmN5f8+/+Jef78TzwWu3whztvfhloUeH5aGZMsRo52Wrm6vpNftPTRFY3z67JcVJ8wUFEsI51tFfkI3n4mugNM39OA9ulnSeYWoHY6Gezws+7O/eRqD2JJDvF42mzMfXcwKRZDaAsAkFPsRv2KwCvHz6GsaQ+iehpaJYRdM8Rg5CSmTi/4Yic+JSXlE32qLgeKorwKvPpfv/vHf73+E/CnQ1e1D9eye4h1jzRgduiomHM1sbV/RS128bb3bQZe6WbG1gOUN7cwlG7jwePTOJBZQ1y1DYZA69ZRkjiDhgkzGLam4VTFuVzaw2X7f4suMoI3bxbxnjrUUoIn9l/H9yuu5fmC45lp62TK/n3UTZxIwZoeLBMUdLYIRLyQjEEy+sHn/87tv0fSUIjq+F8gTDiJ4KbN9P/q1yQHBki74HzSv/c9RKPxE4/DG24fl9V2kKnT8NSUUgoNYy1pg0rk3olF3NTSxz09w/RFE9w1oRCj6oNfxhRF4eC6N1j30H2IosjRv7yJiqnVuP/2dzwPP4z3jfV0LP4B7V47AjDeup4ucwk7kjpciW6mhgUYHkXUJcnsjgFqtk+axpmvbQRRpECzg+6QlcysJWjSP3mfUlJSvpivXR+ysKuDsOkJfGI1g9u1xJwllEezMQ0HmLntTTJHAmyd76LurBlMcRSywlJAvqWAbjmDO1s9bJQE0pH5SxZ8a/PP0fRsQ86eTp9YSU73RqKKBZ0QI4qW25v+RGGkm9uLLmVwbgaLdqxjR/VMXDtGKH78MQTxIzJWigJyEiUWIrD6VbyPPESiqw1NdjqGTAmbtQX1sxeTfPp7BPeqUFkmknfb4ximTv1Ux+DpAQ8/aOhiosnAY1NKSNdq3ve+ShD4dXkueXotv2jp5bS9LTw8uQTXe7oMhn2jrLn3Llp3baNgUhXHfecHWF3pAGT86DqGxx/Llpe6SXj0ZA7tZKSqgFJlC38uuQFdZC8AEfcEzHIcc4UP1WYD3Tk5qOQEaarxKArMsj/DzsFcZqyc/ZnPc0pKymf3tQvo9Xv3ENY6UDT9xDQKGo2Jsro4VXvXklDF+dOpIkvP+h5/nnQaiqLwtifArzsG2O33YolGODU4xK2GLeifvg/FkMZQ+mmYet8kW/CzTZ6DtEVGcKRTNq6OTE0bP+x6ispwN1dW/oznZx7F0n1bKW5rw/rII7guuOBD6ygnEviee56R++8n0d2NtqyMxFmX8VbzQYIjw8xo01EidOGqDJI1I4Gir0fwvgqhPDB9/CS2d3cNcXNrHwsdZh6cVIxZ/dF90S/NTydXr+HKuk6W1zTxeFUpJUYdrbu3s+aevxILh1h0/iVMX7by3T9Oo4Nh1tx/kOGuIIgGouoOKrxrcGzvRDMnyjOWybhG7sKeSGJsSwIC2fk+jB3w7NLpjGvaj6JMQ0sQgzBITFmBeXLm5z3dKSkpn8HXLqAvnLsC//BLRNsaUI0McJw5HXavI5FbzM6cYxnI2MxNu25i/UGJ5oIq9sdi5Oo0HN/XTEV3I1cLT6CL9jGStozYYAc54WfoIZeHYmWM39nJzmrI6O7G8IqFrtlLmJK5kRXDmyiOXs0pk2/hhWkLiag0ZP71LgxHHIGpuPjdusmhEN6n/43ngQdIDg+jr6pCf/G32dKwj+4d68lPy2C+pEPV106/3kbvZhvZpgCOGQq29bfA5jtg2rkw9ypIK3nffsuKwq9b+7ine5gTM+zcWVmA7j3fEOK9QQLrutBkmzHPzUY0jrXaj0+38+xUDecdaGP5rkaubNtFdPULOPOLOP6HP8WWlYN3dJR4NMHetZ00bu9HQSFiVBiYrHC+M4wywUWtN8Tbxqn0qSxkR9so8+gQZQXBHEffKyAoMtsmTWPJroOASKFuFw2+DArSZ6ItsH4ln42UlP/fCZ/mbsEvQ3V1tbJr167PvNyzW5q49sVmskNubtj5COWjvbxYtpBnZpxEEoWwXiE43oFkNyFGEjjbIywYOUCGPExvQktMZSYR12IX+1EBw6KaUVWCEYOMHwU5kYbK0EWmZT3Hb40zry+HwilRikwH6Ne6OGXSrXSYCzl+1wYuXvsa055/Hm0shuexx/A+/AiSz4dxzhys55/Hno4m6l95kfxQjNIEiL19CHo96d/7HvoTV9C8YyvuRx4hc88BtJYExjkqctOGEZARKlfC/GsgdwZxWeYHDd08O+jl4lwXN5fnIr5zoVNJyvjXduFf101SSaJBTQKJNv0Q9do+/ISRJAlJlsdSQV/A2nET6bV6Mbr/yjHbM8gZMZBRNULJbh3ajjCX/uSXXLk1TDLq5AznNazrK2Dxsttwnj7uC203JSXlPwRB2K0oSvWHvfe1a6Gb3D1c1PQcKxp3IqhVrFr0LbYIBpy2IL0Ty/FpBdThKObWRtQDXcyMimSqAwwldfQkc1DJIWRBok/Ix61SEVHUIKnAr0VRVICIFBpPj2cB91fU8+T8VzmiJcrK7lkckXOAN/dewbfG/4VXZh5JHBXLv3MFlfWNaHw+zIsX47z0Elp6O9n9l1tIHxjmqGAEAdBXVWG74EKsJxyP2ukEYOrS5bB0OaO1B+m5/nqCq1vZ4cxFW5Vgkvwq2roX8OUdwRUTfs66mJYbirO5pjDj3T738b4gA48eQPQkaRH72SLUYYyLTBWKKY8UUB7NYtjsp1XXRUvXPkSzhcaJs+hTBJa4zBQKYerqG/FG2wjph3BrY4RVCjIysiDjkhIUJOMUJGLU6ebR7jqBvOHniEuQ5dEjAOaiAI6n4rw5bQYVbbUkY9PREkJKerFoz8E4wXnYPispKf+/+doF9KItjRTWbSFksrKjwE7ACb5Zc2m02rFEgvzUlc74XduJ776XiVk+HlefgZUA5wq7KTTXMqzOZIN6NgddQ4jmTpJRIwcRaJdjiChM1ss0DlfjHjyapH8GyWgpjZn1/HHcZh4JFHDjQIDnaq/hrJK/8MbMBYyYbSx2uTj/tG8RaWpk77U/wNo/RKWiIGZnk3bBRdhWLP/Y2cftEydhe/ElRp/+N8IttyBvFtgwcS7h9BHumno2B6Mqft/xT87STkXIvwAFFb2r6pB3eIgSZ4O6keeTauqoRlGPBftiwc+3knBMwMI8YTJZFgf7MmvQiw+STLp5ZCiKIgBOMMoyFfEEM+JxSqJJ8tUShfY8bGIOq0MW/lV0AnuslaAoxKPNZA0bkPR6dGIQ76ANMe5h6+RpzB7sB0WkQFfDgdFMCm2T0FWkZpFJSfmqfO0CuvGSBWzxrqWu6Ew2lGbQkG7BFPJz4o5NVHW+Sa5lkLn2IfSZXv7BOcgqCduUdXQaQnSqnEASm7wZQ1jF+qCGfjmOQRA4WiVwqsZPwqlBcW0iULaNl3uns7X7WJqG5pIeLOaIkjXUFdYS7jZxa8cfuCHyIzaOq2LQlc7oS69y0muvYI7HUB11JIWXXY5hypRPPVmtIAg4zjgd88IF9P/8F6jqG/nNt25iwGLnur2PsWLoWfRdj9C1+mmC0jVYJRutqmH+pvjZnSzAkIyg04TI09eRZelAre1hjcrDM2oVRwQWcpJnMScPrKTaM429ltUMqvehkl2ok+loFAs+l5qK0mEmly9BI5fy0ME9PGGZhTfHRp7oRRcOEdObCOszKRwMoYnHsI8PkL89C0kcpSM9gyXuDGJAleEF1gWmMHVCCaI2NZNMSspX5WsX0J/f8wjPnL6SZrEUs+zltOijTK/dweg+M0kUclU+VBYPj9uOZcTnZFrBQYzRPIZCCqHBKK9pYzQbYihiAiGaQfnAkSwcnElJ2SvEql4nFLGidhspMndyXuFezijaxe7BqbzUdhz3Hzwfp36EJQVvs6R4A9/a9wrWSJC142bwxIKl+FwmTvQ9Rbq6EXf3GtKL7OhthZ9pBnJNTg7B2+/kezvqCEdj/PnuP7LkpOX4jv02+57ZTVm8GDVJ7hV7eCkJ5f5OFqatRcpvpt4YZFAUGAQMskxBHCbHtDjEjfSZ1iEEZ+GML2XFyCV4lRj/FmM8LgpEJAVlFGSfHiloQnbpwL6EDP8Ipf176B90gaKGRUZihum4Yu0I0iiBvDDpr/g5UDqOwqFmYr5qNITxRIPk6Kejn5CagCAl5av0tQvoGXoVQ0Ie5yoPsEB6gz45QZorxtKSJNv6itkwVMLb0WmEcstJ8w5SHDySHouR1x3rqbHvQ0FBDFay3DOFa2MNJNnK+uk1mEsO0DA6kVsNPyTN7OantQ9w7OBOerwV6DUHsM87SF3eBHb3HMtTTafyVtcilhW8RVFfD+dsG+TlyQt4rupo/K1GTs3+F27TPTTX3IOY1GDUF2N2TcJsKsNsHo/dPhOV6sNvtNk2GuT8A20Y9QaeK81EneHg+Y31TK9PY5xSSo0wyp1EOEp8novTt/FSuYl6jRqLJDMt5EDjySAtpKPKrGKmVEMJbciKQF9iPG1RE4/IEQZ1IU5Hy2WKleWixB8mG9mRpiKpUaGLhLF3dZDolvCH9PjJQHJoERMS6mgXkn4mJvVB1PokvcE01L5eti0+koUJHygiebp91PqymeEaj2F8Kn+ekvJV+toF9Gn5Vm4f+g5aMQGiwjiDgMuiMJim5bF8D7JXy5TwVMRwEI+vnZ9U7qQnK4Yi6ZACleQkDcyR4ngc+7hU52VBdhdTTQneDKXxsPlKJLWRsL6MH04/lrL2GMfuGWR2r5kj74ww3lZP9LgGYgUTiLqX8EjT6YzX9DJH1ccPnn2Qh048kdXlC9HGRzi541lsngiKkCRmbCKS28aAOTm2E4IGwTQdlXUeGtt8NIZyVKLA9tEQv2/rJ1Or5jy1jnufXc9RzrkskYqIKBI342WPeQvZ9rd4yhRBEWzMisS52jNCdSTCTU41LtsIy3QepsXiiFkz2NR7LM0jCwjLafgFmZ1FAh2TrFSUZ7PKE2Z1KERCFJjhSVI6HKJjqJv9YQt6IUKRdoDGmVMoCY1S/vZLbJxoI2Q/nWhYi61gmFk1hUA/zbk5VA1biAJl2lfZY5iLpSgDlTU1GFdKylfpaxfQvW/2UfKQQsaUMOaKCJ1ZBrrzDchTBc7pL6TWX0ZUJdOn2UNlTM+R+zLYF/RRW+JHsNYyBLyoKBQERc7MkskxJdjROZfnXZcjCgZW7NtDm93B/qJFtBZreNj+Dx4hTp7PRJW6kpK2PmZl1LE6rY6DI+NpGj6GXMkG5eO5+bbb+es15/JS7kkMFWRwRdGdaEmADKpBidHhdJps5Qw70qgK7iM/eBvJvtvoVRz8O3EmG9SL0fb4cba0ERFGuTg5jnTFygbVKE87nmDQsZ+IWiCSSHKKW2Rf/gWszjuSZHQVfx96gQFBQkHFC/YMBEVNjs9GnsZEnj5Mf14ar2tEJFFASirc3evGJMdZ1r8e1ZCGCUMTWYYOPbm02WHYEeYHxZPJ9A6y8sV/0pbtRRsxELKfTmtuJQFXM9VbIvQ5M3DSR3RoDmohQn8gSKFmfCrdkpJyGHztAjoDEcLV57IvvYVaYYSG0UF8wij5ToEFGe3MzWqhIWBmWzBJQ7aFOfVmqpscTG630pYTQasrJ18oY0r1s2gMo9Q0HM/juacQMBo57uB2BDnB3M5eBFHNvoL52GOVjGq30KnZQU9iD0yGYqGYaWkC04wH2ZLWy153AVnuiWyvns3ltz5G8nIVG8bNpyNQwZm995FwNVGihRyTm1nWYeLNJnbUr2TjhGlohAN029LY753C5L59TFX6mUc20xIziIoJ7kh/hNedW9EoCgvDcapadSQ7nKjCPoaOeI1p8sPsiSso77nJKCOUgynqYtQwxPbCF9lcmkXEcjwx0zwQdVgioxzV8Bbtw1bWhCegFeO0Z9eyFhMnD6iY68mjZDSL+4fj7IvUs7ZazYDdjyokYwwEaC0eT9i3Ht1gK9sWLGIhbhKKilztAVr8hSyxlWJIdVdMSfnKfe0CupI9CZd6OuksoiwEJ4Zg0D/M3v52dotuMgpqKc1u4gZLEndyIqVZXtwqgVq3mcpOFfriEUoXbQIEdh44jmdKTsJtsWMP+dleMIWCAwM0Wcup7KonpoaGnCKOap5NWD+N3VlaNNEauvzbaA+2Ajpc6hATsltpETRohAoiLhvVq2rYf2wRw5Oz+Gfh96jcuY9B9y6EaJKKwkHGzWplWt4qWnbH2SjPJbM9xCnqbRQjcoRURYZkY4u5hjuznyRL8vAdXxJZyWRHXOKBgiT+cT4ARGkIbUgAjcCRxgyyapfQovZQl7WRQdMAWuNcPNofIlmzERQJU+ggcrebeE8ha+QqLLpBZhnXk93jJbdtCJUi0W8yc+bxFzHXH+G8jgRnSnNY4C1hbXI3B+JaPJE6Dk6Ywco9DtTJAZoKCsgecZEA0oW3UVzz0TrNqDNSg3GlpHzVvnYBPdG1j6Q7iTpvBiASl/sJaD0MSCHsSRfH1V2L0hzEU/QqYsFa/PkJtP3jKe0sIiZ04qyuIR7UsKemhAfmzCRsdFDS/yw5nRXUB5zUK3oIK7QxnoIDMTLEYdaVp3PWQS931QkkhbnExXkMawLsMO9ll2kvBwyNSOYD5Pgz2D57OietepXagxmsixxFeFoae+bNInPTEJNamok351PfJ1B+dAcXzH+Wo1sP0tk7k4mJHKrlCiJClDvT/4lZs54rQkni6RrsxSI21SCVwGBcoCsuMhy10TAs0G2KcMFbWej11yCr9OS4OvEIUziYm4dbZUdMujH2b8TSosUXzkMl2ClSN1ARaKC0fQgVAkF9ku5cBcki0lheQkB+lI3iPhqzc5jpr8Yc0RAPa6lCRV9HLfsmzyYi5BPRtqHN8BLdPRE1MXqCfkrs5Rgq0z5Tz56UlJRD42sX0AeyG5jiexXP5hyE7GPQFC+glj4EFE6S9pCheYIkuTjaCgl3n8NAQQve/G1Ysxqx9S3Ase1m4v4kw2lOVrhdzBl6htNDD6MSAiR0KmKilgcyVrJqdAXukBHf/jiiEOGJSQ5qlV5u6NvOoFREt5iPJzqHnEAJeYF21NEG9PFe5NwJ7J4+jat3vYxe08Ge2EK6p1fzypEr2ZI5RKQriZMg82tbmV32NqXZfmYPlmKJFdMTamSXew0ZbWFGzZPY5NDg82nR1Kuxq7XEJkYwmjSYTVrUZj3FTj2WmJ1tx48nFlITloc4WDoVBDWmQAv5/QcJd6cRSpaQEEaZGdlM1VAzRjmC0RolbYIPU3EQISNGfULNy34XOt9uFobzyQufgCBrianUdKabmBcdoieqkBeKQkKiNbOMPeOGma/tIaRoydTsYVgqY5Y6B30q3ZKSclh87QJ61Dode1Ed9qI2ksn7eD3exLA4laMTSVTyifRIF9KuGmCf1YdLq2JGhwGNegjJNYAvZxO+nE2IQwuY3noiSxpiwDL6WUZUDOHT+9ErLVzUcxAxr48XyqejHgrjq/MQVznYPymHayMTGd96gMzYNjJjQ2iUsZ4rQZWJPn0OQkSE4mJK29opihxgS8lB0voLCcevxVuZiZgWYSBh5zl1HltUi3HG1YSrYEgtEVFVIYnTkdTC+ybA+EgCCHoZvT6CnigG2cacwFYinQpNfcW4KaI43sEEzyYKI11YssJYZwXQ5AeQzUm0SR37AyZqWouwhXOYH85Bo6hIigncxj56zN0023OYkj6BTOUF0sJGGkLLsfT72FVZRWl/G0VNY1PHGpKbKck9ElHQoCuyfYmfgJSUlI/ytQvoe4arCCjnkycMoBYT7DJOpVrey0LNOgBiWMiWypkRKKdFm0HfhA2IGb34Wk9gJJSOd1w9VVlb6M/czOu9M1DaJzC+T8CudaFJFmAX5zPKQk7qgJM6oozN0mclvDPIoDFGnykPoyNBIpBJn0XNoEqLV5dk0BjhYGYQ2RFhRavI+oWzOWnVq2wdZ6Yvuxsx+D2c8R8x4BqPqEi4EiqskkhMSjCSjEMshl5SkOJq5IQaOakgSQqCJENSAUlBkBRIymM/JxWQ5LH5oRQFRYgRFWT2KiUYpTDTA3uZHDxIgXMA/wQdNRk22rQyvsQo6oCB7MEcckM5ZEWymKSoiKhFWjJy0TJMRkJNfGQidsXB9blryRU2I8ugMcdRrEGmNNSy6diF9BWkkdlchIo4nQE/R6aVYahKQ1Cl0i0pKYfD1y6gOyMmQvFqWiUdQVsLmpiaQO0E9oQGcZk96B1REnl9ZKn3Ui3L0ADxJhMezTpWWWawJ3wkrw0cyxLj61Tn7kbO3UPtwASUt2KIQwKSCEaVCauoRy3aMKgt6NUWJEs2guLEGVPj1BXhFAoQeU/gioLcqeDtlmjSuanV1tJ49Dn8ol7hwZyjkenENPIQpoFyznevRBKSPJD9LGtsu/jupLNZbJXp7XoAhQTJqIp12xbR5ZmEwZhPhmgmF4EsQUKl6AjKSdxJHyPyMKG4F5unh5yRQfL8bjxmA/KEAJaJQUqkIioveIbw3oPM3Pg2b2u6GNRUYA9aEREJqxI065I050+jL6MA/c5uFH8mKkHiWxWrOLpgA6KgEEtoqNtYzLTFTdid3RxxMMAWaR7dzlKmNOtwibXEnRMwSVb0lanuiikph8vXbvjcW37wBKZIBqNp+5DUIewj01BL7+9RoU9rpWDuX7FGoljqZpOb9JPQt+BK9rxbxpfMpF9VwEBxkmReD4qixtM0jcHdAomIjKyNsMTRSHqan7dmZPFS65n028vpzswhrtWhkmSqe3qY191Fj70YJWEhfTRJelzBhcCwpoERcZiTYzNxYHlf/Q4Y2/lT7gO41R5KDFqOM/moMMgIpilk6avp7f8XCgqNeyeytvB8GvVOxJHnEdRvkTFwDDNby6gebqF4uAmjp31sujtRjWjNQfb1jL1Or8CbPZH6TAdttgAqtYCIQEgVokst0x4pZzCeR2KSAznXhPlACzZPLwvtnRxb+To6TQxZFlBa57Bnl4Da76Xy7G7iSSv962dzX/VionY7V70aJT35L/Iqj6bAU0rOL+Yg6r527YSUlK+Nb9TwuXarmxqdn2ytn2CoFLR60oQkJmMXOnMr5oyDmPIaIKBj8MUFPGE4gu2TS2ifbGVpt4dbGurRCM3E1Y1k0cz4jmHCAyIdBUbE8TtwVQgwkMdWzWzu10/ntn23s6yth/CEN/lbtJIs9zB2v5+6snFsLyxgZ14u4wa7qGp5G0PrXgbQs8lUQpe5iIUWkZdi61m+fjPPLzwds0VDRK/Crwwzq3cOB509tNHE3yJ60OSjESaREcimfOh45kzewYRZBxiQX8OxOUH60E7mNdkoHVyDKv4aMhCwZ+AtW0x2+iREZynDKhXJyDDDob106YMMOzUghFAnQ/TpBykUcvlWYCH6sAE/cZ4t1PBErol5nR1UB/YxcfprmExeAIZHsmnbMxVNZy8qKUk0q5D+UZnC3GZGXFrUwxE8+Tl4zSGkXg+zE8XoSu2pYJ6Schh97f73pWujZAsjlCczKLI3EMh6BDGnE5UuSTIpolbLdEdyeXv1Ik5pW4d9KnSNc2Dyt7I2LcHl+d1UhsAkTCQsluLRDOBVDRHxxzDGglSnJSjO7GauuhuAtfPTGIwLjAY7mJ/4Me1RLR61maJmiVFrGpKopl8Q6K8Q0ZVIZCT6scebsCf09AeK0NnK2DEum9zW59ibbwMBmrJlVFoFU0RDZbwYn2aUQcMgCfXz9GmthF2Tiew5khOat3Jm00bUgbHUjscpMziuBIejjHTzfGzaDLyaYXbYDrLB7x87PvoQOLT4NFqI1jH94AAz6t0giCg5EmJxISpnBfVOK09XmDlyKMoPtE8SmrMTAHUwG93+8xnp70c3sgut2kRlzgkMmRMMjSgI+Y3Y0vpJGw3SBdQXxFiuTEI1KmBYlEq3pKQcTl+7lMtrT97Ajj434cLNRASZiCSSiNjxi0kiYojRpI5AUo0gRkH46H0TFDBIOiyyGatkQi9rEWWJRDiELuEnw6Ij2zpCmjGG1QgWPajEscAaiIsMBNUMBPR0xrMIjeix+mR0cQVzWMEaVtDHQQEiBRVIeiOWlgNIRJEENaICe8oC1JV4UUQwRRQmdSpM6lCY3KGQM9ZIZtQEBwoFDhQLNBRByGTAFcvAGXVijxnRJB0Yk3oAJBS8mjgDpg56TR0kJQsV/slUBjOxBzzktzZQ1N6GNpHAY7fzxJIVNM7K5zrjb1CLSeSEjnjDcqx9M+ge2kZ/uJlcYzmzXMejVY1to18YJrDwp3i9pexrOILnZixBnxzl17v2MYWZZN0wC7VN95nPaUpKyqf3jUq5HDA08IyzBYIq1IgYRNCIYWyqONGoC1+4kLjVgcZgJqoxg2DkxPUbWNjUgdEXIWpKp37iVDqcSbyJvUxqzGOyIY9iy1TSdbkk5DgDkXaGuoKMCjkE5A4i2l3Y5WaiFoG4HRSnREF6jPJCPwvxI5dAZMRAaNjGyKiNZqGcdmMxMa2J4hE3WXIUrSOXIzds5J5zz8LSNcjMplZm1TvJG/IzcagPEYioNTRnprOtwsCu8mGSTjvjQiWkJy1YowHEURsqaSy4JoQ4oxofXZYuPPoRRgwjJMUEmeFMZrhn4Iw5ERURIwp6cw7hSQW0lSXQ9dUhu9u4sP9f+FUSigyBtlISPWfwXHon5cOPYYlIdJcI1OR08Zz4NwRBZk6oilOaSknUidgmdSOqwhQOhKgpdxBlAE2uORXMU1IOs69dQG/kFEy9Q0y378Shs1KRVU+WboDHm06j2OgnWOFkLccSVJsxxaLc8uZmZh5sINip4M80Yur2UNb5JnW5dnrsDhBHaQ34qA21YhHUGEQ1CTlGTAq/b7v7yMfsB+tQEJMpiAMJiylBIhO6sjMJOtSkpQ+TIQxQSSORuJFWyjkwoYr+fh2iKo3+zAwuefRx1JLMsNVIXa6L7iw9XmshncYcNpRMJFPIZqoqwtGxAN4BP5IgIwOCoMFk9JCXvRedLkR7x1Q0oXTS4+mo/CqMkglNQosmKaKJy5CME0hKDCGzTx1DMceoKq1lz/GVzBN7MSCh6tbheBByBrrZWXov0y1OolqZ1bPdDKbF0EhaKkYsTGzyYBseIKzMRReTiUwPYTF7SPP2oIgT2JxrZ8n41MxEKSmH29cuoJ9tsLKqZx2NpvlcVvwAWlWcB9rOJ1zmoE09hWZhPKpknOLRRi5t/DueiI7nMkuRbfLYzTqZCqKiIL9nMCsRBaMSJ6nSMaAyYTFqUKt0aIV00qUcMvXjyNToMar17ywhETO1ENduJBGpYUlTMyYlTlDUsyVjMnLMhVHsoTK3gUnafZANcpZAcIqDwSYdwREnwT41sYSGhNmGO8uMwWhhiS4AQgBZEYgoRgKSk12yhX7ZQhKRrJifiSoDx1euYdq01+k/eAKjTccgynoEBJKCgqwIxABZSCKb3JgNQ8zM2Udh8R7ajQWcxwNIkpqNbbPZEpGJn9TGkhoDhpiFdH+E8iE3hYFcYsYM0gYHiYtx4o4iJhW0Iu3ciCFUzKjSTFHaCO6ONoR4BbszKojYFVK3E6WkHF5fuxz6T/7xHC3+g5yd/ywhv4U3Q0uJR0SsQR+WoA9rcBRTOMh/39oiqGW0JLF6Eph1ScxTilFtbsI4EkWZFkN74UT0xedwQJmNuSNKbmsX3cp9NPalI45MxxDJwxhcS6F5gOlOHX5lAopYgippBhKI4l6GrDvJkreTFR9BUlSMRsvoCprZW5JL0hXCYR7EbB1Bo0oAEI/r8fvTCfrSCQ1bCPWDUbYQsTuIyzE0SQFRNtMlptGistCDkRBaTJoQF01+kKmuJhqDedyduIERRyaaeIKKkUFK3b1kjg7isveQX7wPm3mEoGLELITZnhjHS0NdBGUBly+NI/fYsIRlonoHuZ4EhUPtZPrGLrAOW4x4s00smNCBTQ7Q8mIWmonLaTtjPypthI17F/NI+SIS6UZeaN7M9Kt+8AU/FSkpKZ/k43LoX7uAvurpW2h5duP7fpdQqYnoTYxa0wgbjVQMtiCJBhSTgYzyAQJZNmrlGqZq9VTvi2F5UiKRq+C/woB9ixPVq30oVoHR0xIkZupwpM3HbpuORTeVweY32d66HW/N+WjidnzW3RwReZ6Fznr2iy5Gp2eQbKqiazSfLFUJg+YC/l3SyXl9Gzh5aCNaoRcAv1zIU8ICegUXFlOQXEsQp2UEla0XxTgMgCwLRIZ1RAaN6LpNiK0SeCVUsoxKAVGEIX0GBxylHEgroGBSFyePewV/zMLrm48luy5EjmaU0aNlyoobcWo8RCXQq8CtOHk4uRJfy1tkjlop69aQ7o2++4dPFkSiOhNRox1UFsoGB5jesg9tJI5okhnKyMTVPozm6IXsLrfhqnqOnVtP403zVA5WVXL97mf4/nW/+SIfiZSUlE/hG3VRlKiApTTIJv0R7B0/B0SF41vf5MVJK1FENWfU7ySkNWAsLqWlJJ0Wz5/ZMdpBkU5LNP+7qCcuxxz8A6FXX8H55wijP/SRnJzA9piKtPvVxHeqCJ67D7f5TRIJHT09k+jvnYJsryfNOwVrcArr02RyIwJVhjr21+gZXvAm+c0zsNRPJNsvkQgV8kjWt9lmPI9FvV5QfMSVJFpZR6EqQbFvBEcyC3M0SH9aOpvLTHRktaE1NTMh7QDFzjZUVR4ABK+AtkNA3SEgdGiw9YUpHe5iuUokeNBMy6Rx5Czt5IzF/+b1kiXErSPMdjUQTqiJSCI6UeaNxNGsSp7Gmc88iDlsIq4R0SaiqFUyroSN4q5ebL5h+tKKuWPKfDoK87gt/69UVXawzTORng4Xk1qaEc0yIX0Uf99CXFXPUZ7mY2S4jVp5HOszSvieLCGIqUmhU1IOl69dC/2uDa9zS9SBpNZwds2LnLB+LT+44ncEdHpOqFlPiVbF8ccfT2luEX958jc8rH6eckMFp6d7SFe6qGMiazSX8N1tAxQ+chuCKOG4dAXeI8YReOoZ9C92I1sUWs7KotY9DzlpQhOIYggHcOWb8PcfiQDY1DJaeonKOoKKC0n+4HRrChBTK+iSAsZkiDTFh6KL0C8XoaDCHu8ivXMbWd5GSM9hpGghO8aPZ1uuFof3dRa412HOCKPPkzDpAgAkFTWjficRfxoBfwbBIROix0POzAPkFISIxtUEwjbS7SN0B3J4ULqUJuskph1sRCUqTD/wFlb/IEmdiWk9JgYcRprKKqktL6bDbiBf3c3f6m8mI+7hz9kX85xqBfmjHiYPtHKC5kV84UW0RxcwbsUNEMjh7YZpPDJxMSqjwJa0GK65S77gJyMlJeXjfKNSLk/f/Sz1HTYs0TAi4DFZkAUBeziIFgXVOxc7E1KcJBIqQYVOrUcQQFYSKHIMkJEUFUIc7D4fgqIQNuuJqrSoZQljRCKutZJUmz+yHioUDGKYdPUBjIIHnyZOstKLqS5AfI9Mjx02T6nmrSNWUtnSwMotfdg1Ocy12IjIB0mo22iPTMedLEVBQh/tI6dnBw5/O7LOwkhZNTvHFSP1bcI52MJIgYv+eRVMUQ1QTAMmQyeCamykx3gwDYNPg1PVy4hLhSwIWFsd/FV/CasL5zD94C5crd1UDtaiUpI0li3j4KTp9KWpSWjGki7mSIKftDzKxZ6H8SiZrPb9kNFoGQAyCqIUZ7HtTjaHvoNBFWR89Tp8uZvYtPUs1meMp3ZcBX/f/xQnf+/3n/mcpqSkfHrfqJSLNiMHbcMQol5Nn8FIQlSRGwxg16kQBBEkhUgigiBImAQtWpUGRVLGRiSURVDGgrsggKAWkBx69NE4NgmMKi2SVoNBjOPo24tWTmApnIvOpsaaYUEKjhBu2kefJ0ZP7pG42jYw3vs8hUvcBKMGtoULUJ/ghrKZ7BsN05z2HMVtJurLjkadCHHc+id4LZSNWmVHEHOYZFvDkdZWmqNHUCccQ3vZqbS/Z1+zOkBRVpA0N+DofRvHU9vYUbWAP808D72gsMzTzIrQy2TrdxB0SQzp1GjcZnLrrax3TWZ14Rwu636KExpfYvNwIQ5thKW5zUTEevoaM/CqMkjKGZhCeqap15Ku6aApsoC3/d9BSmqw+5twjI49rP4OWqpOJuYwk695HpW7CqVgLeNtIUY97dRSwRpjOicfrg9GSkrK1y+glzsyqbQmuaraSKNV5LaaCHOi1v8UUANq0/sX0nz0+mQxATYdRBOQTCKqQ8TNJvxJG7ot/0TpXkVCEFAiMUTAKKopzi8HXTkdRctILO/BZ25lansjeW6Zvf0lZI3byYLWOajD89lgfwiDHw5UHo0tnGTBjvUkkoMoJNgdgSa1k6MyX2R+xkPUhmayKXgiEuUIqJEIklB3YorVM6mzh2GjFnHfBibX72R46lTOVV5herCWg6YyXsm4HHushAG1hTer1QwbRPSREKP7w2waLkJ2mHCaBHqVqWjkAHa8lMtdZAqjYABFgZhfhaNrD8cM/4DosApkAQQQs3UohiT9phmoiTLFtJ6t/rnYJC25tiGsXgPGQJCd6RNQfD0Itrwv4cynpKR8kq9dQA+pPFw7HVptIivqA6wbibKRJHnRDvozd+OxBKnoNFKp5OPIzSEtPx9nYSHOggI0Bj2CWgCVOPYsCnhHt9Hc/FtCnnpy7jWj1MVBUNAp7/T/kEBQGeiZMgdpej/586qpnHY9E/VGVt2+h/6Gi9EsugVrSQYT25vwHphGj1hCTuk2prVWEx+Zx1bhSRQBNs04Gp1WT519K7I8SnVrPl3ZE3hKUDipfy3n69/kO+bt1PhK2Oo9GlST0DMRST+R+vFzkOJ1mBMHkSUf2h1bGbQmeNoyn3/knkNPehl+gw6VrCAAxf09nPbaowjxEJWDAUo86cgDTaAkiM6rZu+ogalpCTKzRhkNGBnYZEPxCSgANjAXRzBnRLFkRtHoFYZGskkkrDiUerLUI/jkVrJGKkmk1yF1VVLgHqSxqITaN55i0mnXHsZPSErK/7++djn027ft4g9hFSe89W8mtOynV5/NzvQ5dKuzEFRBlpZL/P6UlditH53//m+KItHb8yz7N61C92KSpCedUUsZikPP+MTr6HY0oLapiHzbwXBZP9pgNnm9V6OzTWFNjRuJBCXH3sL41n1kD0ZYMzSb+BFqDBmNtDTPonmkkNX5OwjYTiBqOfoD2xcUBX0ihjMyyg+6HuIMz+tEFR2vR+bQNFqOKZmFWlUEKgcq4hRotuD272c4moD39LgfNdt5ackZ5A12cdTW1RhVJuZlnoQpKRDs2ojcuQVdNIzekiR3rhd9WoKRRhP9B5yM6NKoyahkddE83GYXpmSCSf5OJvvbGOcdYChrFm7zVPIGnuHEqY+xTrUcyZmBMP5F+nZfylZBz3MzFnF5w4v8+ju/+MznNSUl5dP5Rl0UHWht5pk1a5iVk0lGQSmemkGul//IUNJBvnwZB7qTpJm0XLqwhPPnFmL6mOFcZUmmt3GUlt2DtO11Ew0lUGkl7AW1mApexZjWQkYiE/UuNTzhRmdPYDnJR+NkCzGtQFb3RMS681nvT8OsUShffC/j2t/EEpBZN7wcYUEAMg7Q1DSH5oGJ7MtuZdDQikN9AWX+MMmkhqhWi0YXI+5KENZpCYhWXN4gN3bcT3Wgjq22Kfyk/Ic0GQvJ8UhUdcSY1BnDkIAo/SRCq9HEPQQsLmqqlpA+WMeElv2oRBd69XQMohWVLKLIMpIgUmLdzRznE0QVM+tGv0NPcsanP/iKzNyt12M7I0lS1PKmvIzxcx/F0nQyL/ebeWjuMsb521lz0grQmj55fSkpKZ/ZNyqg/x8pEGfnI6v5ifYWEjqJvy/9B1PSp7C708udbzWzvmkYh1HDJe8Edot+LJEuSzK9zaO07B6ibc8w0WACjU5F8RQXZTMyyJ+QhlqjIhzuZGRkHW73WryjO9DuS5J2r4ZojhbPSTbS0nvxpoM5mMR0cB7ruy6h0OQju+BtyvxPEZd0nBi7iW9NfYbJrgYONi5gdKgIBQUBAZ0SZR67KTS00zFdIiGqSCR01NctIhBIR0BhkbCZ+cpuVMjU64q513AyA/pM0GlxBK24hsy4hkWI7iER3YqgJFEQ0Opmo9FMRRxLoIz9qxLRq8PIqInJJuRPyrYpSQRBQtaIhEURj6Qg+Ns4f8cdjK6sZJZxHbdyObOnbUaTsPBC3RK2lJbRmZHNQVcvtmknfe5zm5KS8tG+cQE9MRRmw2Or+Jntdgx6I/cuu48yR9n7yuztHuXOt5pZ2zCEXa/honHZjE+o6DkwQiSQQK1TUVw1FsQLJqSh1n70DTHJZACPZzPulx+DP+8iXiwzfJWET0jHrvYgiEnkbcfQ1H061ebH0Rl2M1HVyZDaQp35HEy5Bwil1VNTt5jISA7FShfnCs/iNpqon6pDUosMxosYbj8G72CUyiwDJ+i2Y+58ixG1lg6VwIxYDJ9gZo9/PkPeAkyaTGK6CgbjZmRAlRgiGt+HqB2PSlMAchJBjqLWRjFak5hjAxiUADFLgqSmEJMvF5NsQoOENFJP2LOP3mlmXOWVaKIxRjt68fX0EYr7kJQkPfoc1qYdzROv/YrhydUcOfFFXlSdRjI/iC2vhs5tP2KjVeK1yXP5VfOzXHHZzZ/r3KakpHy8b1S3xViHj9ef+jc3Z/ydDFMG9x3/T3LNuR8oV5Vr4+a5ZSxP6OjaP4Jq/TB1goKYa+So08qonJbxsUH8vdRqCxkZS8m4aCm+rFfpu+468v6VS/tZ2bw1OJNiWzfT57yBTipnV9/ZLLZJaKQ2cpOjZPnuxk0mbaKT6ZXrqK2bxjkjaxg2m6ifokdWCWxRLePiI28h64g4g8/8mPS2Z1EQGcw4H13+FVj7R2nt30qB9ACLLKsZ1M1jzehs/HEzzuQIRbWPYPO2MFA8lUZrK6r8UQrn6tBZ28jp91LaHiZpEakpn8gTo8U4+tQMpO+mJCuTb7UdhVo7GSGnivLhTlThETJ+dA7aLBdSKIH3xRZGazoJiSFqYkE6HDk4OkcYnuIkW93OBnclUwtkppndNHklVJLEBmMeV8jy2FgFKSkpX5lP1UIXBGEpcAegAv6pKMofPqLcTGAbcIaiKM983Do/bwv9zX2v8aM9N1BqK+UfS+/BZXC9+54sK/S3jKVTWvcME/HHUWtFiia70BabeapnmNcbhrDo1Hx7fhEXLSjGbvzgHZ4fp9sTZtu9jzHhX7exN72c382/iLLCBPOyn2GacS/db19HzJ/DjBPeoGpwL4aOGpIiIAjsnWTFZ9MgyAqKAIqsoTZ6EsvMZ5LXvgZ9z98Q5VFCySX4kuch40RGQeXUI6YbqXN7MA8+zgzjsySB3YFiHKsDYLfiXp6Lr2AAY6YHxAR6SU9lU5y04SFa9UY0iRgFkoQHG3vS5jPx3N+RlVaMHJfw/LuRyL5hkMMIajNKMoauSI395Gq0uWYiB914X2hh1eB++joPcGbzWoKnlzJJtYO/yBcxf+6LmIen8lhLNVsnjmPUbGbfeC2G4tmf+fympKR8vC+UchEEQQU0AccAPcBO4CxFUeo+pNwbQBR44MsK6P3Bfv6y+y/8Yu4vsGgtyLLCQOsoLbuHaa0ZIuyPo9aIFE4eS6cUTnKi0f2nJV7X5+eudc28emAAs07NBfMKuXhBCWmmjw7svkiCVw/083xNLzs6xsZYuSxcx8lrHkC3YCFFf7sLWa3hhZ5a/AdvIvHGGSDIFB39G6Z09JM5HKO1OA1ZiRPTaRlyqdDE0kno3aT5opS3hjCHJQJGF57809BnHo85fTwtI928tmkdan865kgBUgIKJukodawhf/s9WNIihOJq6qaa8Ls0aLRZCMaJeHvdzN+3AbOU4Fank66yYxjXXU5O7x6WmhuxBZvB6IRZl8OsS1EMDkJb+hh9uQ0UD5KnD9FagaDWosk2YJqbi268g/DwCJf84TFu2vgg/fMXsDj/aR4Rz8Q4rpk0m5vmzTfwVm6U9eOmcWfHc5z+7Zs+8/lNSUn5eF80oM8FfqUoynHvvL4BQFGU3/9Xue8DCWAm8PKXFdABFFmhv8031hKvGSLsi6PSiBRNclI6I4Oiya73BfEP0zgQ4K9rm3nlQD8GjYrz5xZx6cJinOaxWXfiSZm3G4d4fk8vb9UPEZdkStJNnDItlxOn5pKfZsT71NMM/PKXmBcvJu/22xC0WpKyzAubnmHoKStaRxfyEW9wxsEtmIJB9o0vIWCVsbWcS1Myh2N5FMfoFhImB70TK+kyD5FIjo4dT8VFsP0U+g5UIcV06DJqyZzwApnbuzGvFpGtAuZzHWQH2tDGIjxjNXOH3caFvgAX+fwMGW00LP4JleWn8fIzL9Pd3c2yZcuYPWsWdG6BzXdA82rQGGHauTD3KkJeDYOrNhETeknodhOJ9CFnGUhafCQMI5ikibzwcjmnrFuNO6uKRUe8xjbtLGrNTionbCJjx/Xcl/Ty9LxjOLXjDe6+6Mef6/ympKR8tC8a0E8DliqKcsk7r88DZiuKcvV7yuQCjwOLgfv5iIAuCMJlwGUABQUFMzo7Oz/zznQccPP2ow2E3gnihZOc77bEtfrPfkmgeTDAXetaeGlfHzq1inNmFxBNSry8v5/RcAKnScuKKTmcMj2Xybk2BOH9I617HnuMwZt/g+XYY8m99c8ImrHeNLVbG3n7oV5sRZuIVm3m5H2NGFSwatbZhJr6+Hb/S6g0BjjyRzD7CuJyAJ9vP0MDm2jeJjFwYAZS3Iw5Zy+uiS9hDnVjf0iDthcic/Ip/e2fsOZUIcT8sPY3KDvuQxYEVIpMYurZaI7/M75IkkcffRSPx8Mpp5xCZWUFsVg/kUj32GNkD5H+TUSjPUT0IgnN+3PeomRAPSyjcmsRsrMJO5swPH8WdbWNTPT0YDrVSIbYwX3y6cyb92/S2o/n7o5x7JhZiZ4oq2eU4sgb95nPSUpKykf7ohdF/3uuCHinJ9x73A78RFEU6b8D3vsWUpR7gXthrIX+Kbb9AZY0PRlFVsqqx1rinyeIv1d5poU7zpzGNUeXc/faFh7Y3I5GJXLsxCxOmZbLgnIXGtVHX9xLO+cclESCoT/cQt9Prifnj7cgqNVMnDuO4LCGXa9CumOYhgltVO/zcNbbdyMLIomJy+ibXM1oogX/jmMJB4fwthyFp/E4pJgFZ9Ewk46OkVM0h8BD/YQfHEI2i7gvSxKf2squxlNQtdhxOKqxTZxOWuldWHa/RHLCCYRLpjDYuYrt218mzTnK1GlGRn1v8fb6PhRFerfugqBBb83B4JxLxqgXQ2cD+lAYQ9o0ZE4nuL8CTa4B2bOakVUvEb4RLNMlRv1G7H2jdA9Nocq5CycBosE0gun7mNp6FEMjA+wonsArzz/Aud+95Qudn5SUlE/v00TDHiD/Pa/zgL7/KlMNPPlOMHcBxwuCkFQU5YVDUcn3cuaaOf47VYd6tZSmm/nLGVO54fhK9Brx3X7rn6pOF14IySRDf74VQaMm+3e/Q1CpmLW8GG9/iLa9J+Ncaqa+8lEyhqErX03ItB16t6NVFxHsPIWePZXEw2ryKm3MXllGVrGNWHMzfVfcQLS2FusJJ5D5sxtR2a3s37+G3TVPo9f3IkkHcLvfpBUgC/DsgrE0Pzk5oFalodUWYjBMw2BYiUGfj8GQj8FQgE6Xydilj3dEfbDrQdj2dwheizWzktHh5UQ1R+FcaGd0+DH89h2U59lgO3T1aZGdAvm6AfrdeeiL9zFXJ7F72APFUKvWoijKB77VpKSkfDk+TcpFzdhF0aOBXsYuip6tKErtR5T/F19yDv1/lfsf/2D49juwnXoK2TffjCCKJGISz/15N/7hCAsvHCQursVsHo/RUEXv/mz2vTVCxB8nb7yDWStKyC61oSSTjDz4IO47/4poNpP1q19hPe7Y920rHA7z+uuvs3//frKzzSxaVIhG28/ISJjt21vRanI49dQrcDqzP/uOJGOw/2nYcie4m5CETPyRE2hlB75jByhquJLOB19gUJ/G3GP7iarjrBKPonrmi6TXn8ev+nI4MH8iJaEefp+Xx/S58w/REU5JSfm4lMsndhRWFCUJXA2sBuqBpxVFqRUE4QpBEK44tFX9enNdcQWuK6/E9+xzDPz6JhRFQaNTcfx3qlBpRHY+m09pwR0EO0/ltdu1bHu+n7RsEydfN50Tvz+N7FIbsbZ2Os45h+Fb/4L5qKMoefmlDwRzAKPRyCmnnMLZZ59NMCjw5JN17NyRyauvxNDrZnPuuT/5fMEcQK2D6efBldvhrCcRc4tx6B8gyxMEIKQdJlRipHKkjX3BKsbH2lFHROSYibBrHxPkbApHBqh1lLJl7b+/yCFNSUn5DD5VAlpRlFeBV//rd//4iLIXfvFqfX25vns1SiLByH33IWg0ZN74UyxpepZdUcULt9Xw0E+3oMgKOeV2jr14IrkVDgAUScLz8CMM3347ol5Pzq1/xnr88Z+YrqioqOCqq65izZo11NTUUFFRwWmnnYZW+9n6138oUYRxyxDGLYOd/yT7ueto7yvA79xJXnYaun1d1LhdnJgPedp+hkYzULnqWCSqqHf7SOapGTGr8Ho8ONLSvnh9UlJSPtbX7k7R/3WCIJD+wx+gJBJ4/vUvBLWajJ/8mOxSG0sunEDT9gGmHJ1P7jjHu8E63tlJ309vJLJ7N+ajjiLr179Ck5Hxqbep1+tZuXIlRxxxBFarFfHLuENz+gWoNt+Bo1tmcHYHmWmnERUb0A2M4C7IpETsZNPgRLIy20lPa8A3koY2maDWNZ6Xn3+B8y6+6NDXKSUl5X1SAf1LIAgCGT/58X+CukZD+g9/QHl1JuXVme+WU2QZ7+NPMHTrrQhqNdl/+D22E0/83BcR7Xb7IdqDD6HSwPzvU/jiTxjERjijk0ROOtWDTWyKT2cZb/JGbCFIahKuWlzDJxP2DLLbNZEJG59HkiRUqtQE0ikpX6bUYBtfEkEQyPzZjdjPOIOR++7D/de73vd+vKeXrm9fxOBvfoOxupqSl17EftJJ/9s9Qqaeg8XmRD+oJpCxDWPGOPKDQ6wJl6FTEuQJAwSDTkLp+1ksgmXYT0hrRLTG2b5xw+GufUrKN14qoH+JBEEg65e/wHbqKbj/9jfcf/87iqLgfepp2leuJHrwINm/uZn8e+9Bk5V1uKv7yTR6mPddsoYDxOzdaLPGuo8KfTIhjORre+kbKiSp91Bp6SE4BIIi05FezNpN2w9z5VNSvvlSAf1LJogi2TfdhO3ElQzfcSftK09k4Je/RD+lipIXV2E/7bT/7Vb5f5txITnS2AXXUFkLssHGwoEW3k5OYWqiAa97bD5RS1oLsbiFTJ+H3a7JDEaTdHd0HMaKp6R886UC+ldAUKnI/t3vsC5fTry3l6xf/oKCBx5Ak/vBYX//5+nMGKZfiXkkid+5EW3GZCrdzbwhTcMh+UlP+EhGrITS93OUKo5t2Meg2Yld5+WtV1853LVPSflGSwX0r4igUpHzpz9SsXULjrPO+nq1yv/brEvJCkA8fQTyctFKUSJDaSQUFbnafvo92URtbczSjeIfHFvEnZFJw4CHcDh8eOuekvINlgroXyFBEBB1usNdjS/O4CCj4FsABIubQBCZ39fKTnkc46UW3IOlICiU2fvxh3TYwgH2uSYyIJtYv+b1w1z5lJRvrlRAT/lcDHN+gtUvESjdj8pRzOSRdjZLMyiR+tAERZSkjoSrlqmqMI7hUTrs2TiFYfbuP4Asy4e7+ikp30ipgJ7y+ZjTyTRMJZEZQUpPR+/vJhgYD0CGMMCo30nIeYAjVFECgyCLIol0Ox5JTe2+fYe58ikp30ypgJ7yuWVM/zkA4YIWQGFG/wDNci5lSgf9A6XImgizLCMER0V0iTi16ePpSpjZsPatw1vxlJRvqFRAT/nc9OkzsEp2wgv7QGOi1NtFS3Im05QmvCOZIIvonC3kKRJOt4cWZzZOIciwP4DbPfyV1DHp9dJ2yim0Ll+O+557SfT3fyXbTUk5HFIBPeULycw/g7gL4mkuVO5GiFWjFmSMyQCxiI1Q+l4WagNEhyRiGi3WNAtBRfOVdGGUgiG6L7uceEsrKrOF4dtuo2Xx0XRe+G1Gn3seKRj60uuQkvJVSgX0lC8ko+hcACKVgygxP+mjanyKlXG00uMuJG4aYKEuSGhYQJRlDrpy6ImZaWprJx6Pf2n1kmMxeq6+mmhdHbm330bRk09QumY1rquuItHXR/9Pf0rzggX0Xvcjghs3oSSTX1pdUlK+KqmAnvKF6PU52AzjSRw5CoDD082IVM2R4n729k4BINfRhVFScHq9NLsyKdTpkRDYvnnTl1InJZmk99prCW/bRs7vfotl8WIAtAUFpF99FaWrX6fw8cexnXQiwQ0b6L70UpqPOorBW/5ItLHxS6lTSspXIRXQU76wjNxTiWeKRC12kkO1qBPVWIUwyTDIMROR9APMVodQhmP4jBasBghGZbZv3conzZj1WSmyTP/Pf0HwzbfI/OlPsZ144gfKCIKAcfo0sn/1K8o3bST3jjswVE3B88gjtJ94Em0nnsTIAw+SGBo6pHVLOTRikSTbXmilfst/z4SZkgroKV9YRsZSAKKTEkieFpTEBJKKhtliHR2efMKOBhaqIwQHx4L3nnQLCW0hwXiC9ra2Q1YPRVEYuuUWfM8/j+vqq0k7/7xPXEbUarEedyz5d99F+cYNZP78Zwg6HUN//CMti46i65JL8b30EnLqDtfDTpEVGrb289gvtrL79U7WPdJAb6P3cFfrf0oqoKd8YXp9DjbrNJIz/AiKjDzSQ0yu4ljVLt7unQOixFTbAEIUbIEArc5MSgUDSEnWrT50d46O/OMfeB56GMd55+G66srPvLza4SDtnHMofvopSl59FedllxJva6PvRz+mecFC+q6/gdC2bSipG6M+UiTgp7t2P3tef4mNTzyEf/jQfMsZ7grw3J9389ZD9VhdBk764TRsGUbW3F9L2P/lXYv5uklNcJFySGRknkBzxR4klY5w7x5i2dXk63fjHnWiSBpkVz2VgxPoGQ4yUJwFYjNKTEX34BB+vx+r1fqFtu957DGG77gT24krybzh+i88Vo6upJiM73+f9GuuIbxrF74XXyTw2uv4XngBdVYWthUrsJ24El1Z2RfaztdVIhrF3dOJu7uTke5OhrvGnkOj728xH3hrNcu/fz0Fk6o+13aiwQTbVrVSu6kPg1nD4vMrGT8nC0EUOO7SSTxzyy7eeKCWFddMRRS/xuMjHSLCoc5hflrV1dXKrl27Dsu2Uw69aLSfzVsWYL3NgbZbT/px15JjuJA/Jk7HMaWbcssgm7dcx0NGG/45eRxdv5tFPhe90Rqqp05h+cmnfO5t+156ib4f/Rjz4sXk3XE7gkZzCPfsP+RolODatYyuWkVo02aQJHTjxmGoqiLzV79E/AbOyCQlk3j7enB3d+Lu7sLd3YG7uxPf4MC7ZdRaHc68Alz5hbgKCsee8wuJR6Os+vNv8Pb3cuS5FzH9+E8/G5csK9Rt6mPbqlbiEYmqRXnMXFGMzvD+Nmjd5j7WPdLArBXFzDyh+JDu+5dGSoIUA63pcy0uCMJuRVGqP+y9VAs95ZDQ67OxWqeRmFKHuTlEeHiYeG4Jy9S7+EP/CkpcjzHP5OHeUTv6WJR2ZxYr+30MRgPsO3CApStWolZ/9o9jYN06+q6/AeOsWeT85VYiiQQ+txu/34/P53v3WaVSYbVa331YLBasVitGo/FTBxlRr8d6/PFYjz+epNvN6DPPMvDE3XgG6wis2EzpMy+hMho/8z78L1BkGd/w0Hta3B2MdHfi6etFlsa6dAqiSFpOHpkl5Uw6cgnOd4K3LSMTUfzwP2bn/PZWXrv7Nt5++J8MtDZz7OXfRaPTf2xd+lt9bHiyEXd3kNwKOwvPqMCZa/7QspXzsult8rLz5Xayy+zkjXN8sQPB2LWY4UCMdIvu0I2KGvVD61vQ+Do0r4E5V8KRPzo0636PVEBPOWQyM0+gbfIeeEbLaPdO0rPnMkl8nC53PooCDmc76aOlJNyj9GRmMCQ2U+KYRFO8iwP79jJtxoc2Ot4Vj8ffF6TddXX0v/kW0WVLiRcU4L/1VhKJxPuWEUURq9WKLMsEAoEP9Kr570D/YQ+TyfSBibdFsxn/trcZuSxMMlshEOzA//O5TL7uJXTZBYfmgH7JuusOULdh7TtBvItELPrue9b0TFz5BZTMmPVui9uRk4f6M3770RqMrPzhDexY9QybnnqEkd5uTrz2p9gyPjhDV8gXY+vzrTRuG8Bk13HsJRMpm5HxsUFVEASOPGscw50B1txfyxk3zsRk+/wjmkqywo+e2cdzNb24zDoWlDlZUJ7OwnIXmdaP/0P0Ad6OsQDe9Bp0bAY5AQYHStmxCPkzP3cdP04q5ZJyyPxf2sX5UwsRJY+i+UvJtt7IzcnzKZpeS7Eg8PSu7/C2U8vQtAJO2L+ZM0PlbAmuxZHm5LyLL3k3WL83cP/fz5FI5P0bVBQMiQSO/HxsaWnYbDZsNhtWq/Xdn98bjCVJIhQK4ff7P/IRCASQJOl9mxFFEYvF8m6r3mqxIL/5JpryNxDKIxzwX8jMxKvEXAOovALFxdeQP/VKRPF/s70kyxLbnn2Krc8+gd5kJqOoGFd+Ec78/0uXFKA1HPpvGm17dvLqnX9GEEWWf+8nFFZNBUCSZA6s62HHy+1ICZmpxxQwY2khWv2nP34jvUGe+cMuMktsrPze58unJyWZHz69jxf39XH27AJCsSSbW9y4g2MXXSsyzSwoGwvus0vSMGr/q36yBL27ofE1aHodhurGfu+qgIqlMG4ZEedknn1hFVOnTmXSpEmfuY7w8SmXVEBPOaS271iBcE8bxq1qVBVLKZ/yLHsFFw9kz+bU8pcZ2fxTfhzJJLk4i/H9nXy3IYhb6aBb+8HPoV6v/9AgbQiFCNz4M4wClD722MfPxxr1w8FnwJI99p/qE75Cy7JMOBwmEAh8ZND3ud1kFe6nuGg/TzScwptdi8gUA1yYsYGC3K3onX5IuLCnX0pW5jLS0tIwGAxf9NAeEqFRL6/+9c90HdzHhCMWs+TiK9HoP2PL8wvwDvSx6k+/wdPbwxHnXEhm2SI2Pt2Mtz9EwUQnC08vx575+f6Y1G/pY+3DDcxcXsys5Z8tn56QZL7/5F5eOdDPj5eO48pFYxe7ZVmhYSDAxuZhNrW42dHuIZaU0agEphc4WFhsZaG+lUkjr6NqXg1hNwgqKJwH45aNfeacpQAMDg7y5JNP4vP5WL58OdOnT/9c+5kK6Clfma6uB+l6/rc4/65hKHcW02dH0GvWcrL2Rn4w9y7Sas/njN5q9NN1xC06Ltm6hUWOat5oepyc0jJmHncC6VnZWK1WdB8yGUiir4+Oc85FiccpfPQRdMUf8R/X3wfb/wG7/gUx39jvihbCcb+D7M/X4wJg6I472F/zMMIFo2zqnY1fuIpco8xfd/gpt0osF+rQ6GrJK6/FaPLh9zvp6JhGLFqMw+H40IfNZkP1FVxQ7a47wCt3/JFYKMTii69g0qJjDsvMWfFohFfu+BvdjUZU2nFYnDoWnjGOosnOL1QfRVF466F6GrcPsPJ7U8kfn/bp6pOU+e4TNayuHeTG4yu59IgS4rKMVvxgr+5oQmJXbRMba/axqStGbdQFgE0IMd/uZUF5BgvnzCE/J/t9y9XW1vLCCy+g0+k4/fTTKSj4/Gm51EXRlK9MRuYyWsp/gywKiJEAwX47lsIEC6JtBCMOzK6DTO+dQfNwmKF0F50WDdJIlCMWHMWOV/7N2w0HOOrCy3DNP/ID606OjNB10cXIgQCFjzz84cF8sA62/BUO/BsUCSacCHOvhv69sO53cM8RMO1cWPxzsGR+pn3zPPoYL299i6yLwvT7i7BqziVz9ADhoTDXHrmIP6/vp37SMfwuZzad134f/1wLthUxqqreRJLG4fUcyeBgnMbGxveldQRBwGq1fmTAN5k+X2+I/6PIMttf+Ddbnn4Me3YOp954M+kFRV9onZ8kFnfT3nY7ZnMlLtdi9PqxACclZPavG2SodwZqvUQ8tAUlNoQj86df+I/L/+XThzr8vHF/LWf8bNYn5tNjSYmrHqvhzfohfrF8AqfOzufKuk5eHhrlF2U5XJzrQlAU6NsDTa+hb3ydBYMHWACQVoK7+EQ2649gkzeHTa0eXt0RhR01FDmNLCh3Mb/USaKnlprtm8nLy+P000//wl10P/YYpFroKYfalq1Ho//tIHKfC0PBZCqnPM0uxvFaSQELs2uo2/Qz7hJMjC4qZHZ7Lee0RVi2bBnR/ARr7rmTgZYmiqbO4JhLrsKangGAFAjQecEFxNvaKbj/nxhnzPjPBhUFOjbC5juh5Q3QGGHaeTD3SnAU/adcZBQ2/Am23wNqHSy8dqy3geaTUw49q17m16/t5Ojlr6FTxdlVcwLapBbSM4mEQhgjIRi3mAf2+Dh1eh43VYp0n3MushxDumIc3qpeEkkvGRnHU1z0PSQpHa/Xi9frZXR09N2fvV4vwWDwfdueMmUKy5cvR/M5umOG/T5eu+tWOvbVMH7+kRxz6VVfSn78vRIJL7trziYUagbG4ovFMhG1NI+m9QWMtGdSMi2D+aeWMdJTxyt3/hEBgRO+92OKpny+NMR7jfS9k08vsrLy+9M+Mp8eTUh859HdrGsc5uYTJ1I63sV36zsZiCeoMunZE4yyLNHBbbW/xu5rA0GE/DkwbilULANX+ftSeIqi0DocGkvPNLvZ2jZCOC4hoFBogeXVZRw5LoNKm4LRakNUfb72dCrlkvKVam+/i6F77sD6gpqe6nOYl/UygqmX6x1ncGLV8xhrruIs9ziMc01YCLOipoZzdEdT9JM5KILM3tWvsOmJhwGYf8Z5TDlqCb2XXUF4717y774L85HvtN6lJNS9AFvuhP59YEqHWZfDzIvB+DFft0daYc3PofEVsBfAMTfBhJM+kF/3eDw0NTWxcctennTb+Pa0h6lwtOIduYwupvCo2cagUYspEuOYhp1k+d30W6tZPSRwwdxCfjrVQsdp30IOBNDNrkL52Uy6+x5ClmNkZ51KcfE177Zc3ysej9M75Kapa4C2tg4GW/aTk5PDGWecgc1m+9Tnoaehllfu+CORgJ+jLriMqiVLv/QUSzIZoGbPuYRCTUyp+ic6XSbdna/R3fYagqEJQVBQixlkZh9DuutoHI45+Ic8rLr1t4x0d7HgrPOZufLUL1zPhq39vPVQPdXHFzF7ZckH3o8mJC59eBcbm93cdNJEutK13N01RLFBx13Gbqa9eD73Zi7n5pIryFLC3OPwMqNy4cd/rt5jcHCQx554khavhLFkOq0hDfu6R8k3d3HllAeRIos5+1t/+Fz7lgroKV+paGyA7U8tION3GgZzZlOZ7yazeCO3cw4T5q/F2XMk1zWeRKgsSVdpPudveY2TQuMptuZimp2FqTqLUHSUN+//G+17dpGm0jKhvo3xN/8W2/ITIBaEPY/A1r+Brwuc5TDvaqg681O1tt/V9ja8/lMYqoWCuUjH/pbupJOmpiaamppwu920SmlsixdyxrjnOapoI23tl3GPdTE9Lg15PonLoyqyBDc3WO1MadtH2VA3e6IV7BdsnJSTxvdmZxP7ycUoA31oiwrJfvhuukcfo7f3cUDA6DgTj3AG7R417e4QHe4Q7e4QI6H/3M6eL3pZpG1HrdFwzPKTmTdl3MfuliLL7HzpOTY9+TC2jEyWf/96MotLP9/J/AySyRB7912I37+fKtMZ2LpGqOmezp6WQkRRYMYSkcxpA3h8GxgZ2YgsR1CpTKSlLcRhW8jeVY00bdpJxZwFHPed76HVf7ELyW89VEfDtgFWfncq+RP+E4gjcYlLHt7JltYRfri8klXaBAeDEc7LcfIrWwjTA0eDJQumX0CNbTKXe6z0SyI/tfi4Qj+KqCTHerQoEsjJdx7/+bm2P8ILdWF0Kjhjgop8s4QiJ6jzDdHv2IAvZqHLfSE/uODqz7VfqYCe8pXbuH4W9uuCBMQSsqctoyjzRtYzi/bJCqX6AG9s/REvmwT655awuHEXRQMxvpezkESbD1QChkkujDMz2POXG9g90EVCq2beCccyM9ONquZfEB2Fgrkw75qxngQfcgELQJFkQjsHEFQixqkZCJr3lwuHgrS8+S+a9u+kRcomih5RFMnMymftkJPtQRXLM7dx8pTH2eVfyu3WS7ApApdrJM7oeBRjc4KENJ6E4SnOnvIDLCNRpnc0sTNRRp3s4IiImrmSFiE6SCDYT78eeqdNxZccYl7GC8zN2Uk0qeP1jqPZ6zmO3LQ0ip0mitNNFDlN5NoN7Or08NaeZtKHdmMW4nQYKqiuruaEqizKMizv259IwM/rf7uNtpqxwHjs5deg+wpudpJ8nezb+228yU4m1wcJ905gfeBKgkknFfr1zLM8hEnlBQSw5SE5CvC6zLjNEdz0EJP9gIhKKqB7ZwQxVs7x3/k9jqycz12nRFzimT/sIhKIc8aNszBZNYS8A1z02EF29sdZWZXghcx8jEqc2wYe47i+NRD64NgzPpWZH477Ma+kH8mSkS3c2fB70pL+D5STEXiL+WxmJvn0cTovYxEiJFQaDpZU4snqoW64kvsPnsU1ZREuuuCSz7VfqYCe8pVraPgFoT88iWa3ieDxv2Ga9AOCJoEn8mYxsXQL8W0/5jv+HNRHplEa7GXmwXpei4/HioVTRQNLZBVmRaA3GaVeP0S+6hkWsh41MnXmuTRUXEYofRomnRqzToVJp37n53eetWq0o1H8zzST6BubmUjUS5jGRYjmh2n2BGnqHaV7yIuigFGvpVDlY1LobZKSxNXxa+nGygp9CysW3k0j47mFnzHtwDYuaHqGoxwW4sqFDGqyqbVIzPAmiNu38qOJVUT8sKihhu2JfBrlbC5INlAaySAg5QKgEEW2xlGXZpM3IUKa7hFiwbfRal0UFV1Nbs4ZiKL2A8e0c3CUJ57+N9GRXhqldLYnCijLtHL85GxOmJyNydfLy7ffQtjn5cjzL2HqsSd8uSkWbwfUv4xcv4r91gZG0jSUdzlo7L+Opq5MHNlGFp01jpzs2FhZT/vYs7f9P69DQyhAwKzC7dQx7DIQNI3VWQwKODWV5Bcux5a1BNFWCB+Xd44FwNfzvoen18+/ty0hQ9/J0dbfcHHse9Qo5ZSUjHKgvIrF3p3cPvQUGUYTjLSg+HpY7ZzPm2lzmRRsZk6okXEWE2RV8Zq5ih/IlZi0Rv5eqGe2RQeiGkQ14ViSZ195g9b2TmZMm8qyZUtRq7UEPA3s3nk5CW0/j3WczgHmI2aJLJWGuHnlBZ/rsKcCespXLhzuZM9fl5D2oJrOqWcwK6OBtLTX+Kv2LCbMeQNn4+lc0LkQ3QSFwRwX5297E1UygWC0IzkK8feEmDPSz2xLHholD4gwqGrhb5KBdYoTWfjobn4q4Cy0XISOEAp/l9zYhH4qtBIeMUpAHLtByZiIoo2IiNE8BKkMUVDTqhvkVb0ekxDhJuN9MGcYv9rC4/0XcXHDKo6ij7h0Hp3amfyrOMmqPDMJUY1Oklk4LFHa0caThZkYDUmWHdjGplgxXZKdm3O2cKK0nb5+Mz3xyXTHpxCWx9IANv0oeWV1aMrfJqppR6/LpaTo+5ga7Xgfewy1Kx3TzGq0JSVoiop4e/duNm/ejM6ewX7NBLb1hFAUSIt7mMgg3zlrKfNmfvRNK0o8jn/1aryPPoYcDuG66mosxx376YL/cCPUvwh1L8LAfmTg4LQ8hi1R0ricPa/OJRZOMn1ZIdVLi1BpPmFA13jonSDf8W7Aj/obGUq24DYEGbVrUEQBdULG5U3iilhxakrpEisYiqgwxD0Yo4MYwj0YE24MRNERGxtGVlCBNYeG+LG82rGM11wBWpNa1FPsxLJM/CQrxtmqLnS1L6Gte4ONpon8vuQy9pnHoUMixthnLC3pZ7Z3L/N8e5nlO4AgwHZrFZZMO9PzZEZiBjZtUiOFEyyo6KfENIDG48abTHKgUMNGZSEvR1cStNgBmBCsY1K4iTvP+NUnH+8PkQroKYfF2y9MJuOGJMPpU6iYt4Js/ZW8wmKU6h7Souk8tPtK9rlidEwv5b7oRrJcc9izp4a+vn5UisQ4oZVpui7yK88gHDmScG0AkjIRY4Q9XevoVbmZ9q3zSJ8whXBcIhhLkhwKk7OxBykQYF+yjTrcqI2gEhQkRSAgmymWnCyRszErBtoTMr0JP0njEKt1erbEHVQo/VTld7Ki8CkUfZKyOhU2vwo5eiz12lN4sETHKzlaQOGE4c2c6F7HBttUVqUvxavTY0hIpMXDeFUqTj+4nQ2j2QzIFv52zgyOq3Ti/sNPiaz9N7G0TPwVR9GfKKQ3UEhS0WHK2k9u1ROI9hE0fWB5QY2uVkBrlDC44ugcCVRGPS3OSlYbp6MXk8xL7CYe9RMxphGUVYiCgk2vItemI9eux6oTQZGRo1ES3V0k+npR4jFEo5FE3MjgRgndlJlkXv8TDJMnv/8kKgoM7B8L4PUvgfudGZ3yZqGMP4E6Ux0D3rdIDl5Ey/q5pBdYWHx+JSGnhqcGPMyzm5lvN3+ubwqJSIgN995EwPMmzokqZGeABAk62qbR0zvxI5cTBAW1OoFGk0CtiSGIcRp8RXjiNkJZVmw2P6dGVlEx2k9GOEi9pZg/l5/LbstUnMowp/A0C1iPm3QamECDMIkGpZJhYazHlSUZYNnwJo4b2UxhoB8xrpDNMA7G0jABlZ67JnyL9Y5p7GcqsqDCFe8nK74FJbSF4dgQizRF3Hn2S5/5mIztXyqgpxwGe2suQvjpZmKjWTgW30Q+F9CnsVNTVIwj/yB9m37Jb2MmwotzOWV0C7cXmpDevIXhuEKNPJUDuilEEjIWi4WpU6cyZfxkdG1xQtv6SY5EiROjdXQPsQKZ0qOPoHN9PcP+GD2ih5g4NqaLT9bTL1vokW30y1YMQpIC0UueOEo2cTIEPWHZzJNSOiOKmjzLKKFxTk6zPMEczWbUB2xUD+eyT3c5/yzO5PUcNaIgsDLNyvXluRQYdWMXxPr2kmx+izeaBZ7XL2RtlpagRgRFoczdj69TIuwVefDCmRxRkcHIP//J0J//jNYiYT9qIjpbHMnThBgP49mvYyRPQ+BEiaQLkBXUXgHRLaD2gZkkNl2cSNLEG8nFjMacLI2/zRSpEVmlIarWERE0RBQRGRGVKGKUE+hDAbTJBKLZgirNiWCyIPX3IqudDO6yEulNYFq8mLRzz0UdH0Bp2zg2BklwGAUNZE5GyZ8HuTNRdHZ6u5/A69lGqHc2of6plFS5yB5nZ7M3yOseH3VmFR6tgD3dyCV56ZyS6cCo+mxTMEhSlF2vP8T+dU9izrMxLFbi8ejIzm4hL7eRZNyFnHQhK0YkSYska0lKGpJJNYmkimhcRadHiyDL6MUkauVjxrIXBfRqFWpBQJBlZElClpKIchxRSSAIMqIKNEgIgIKAjEBC0BBDTYcjm5bsXFqc+cQEPdakj6L+AUqHOnGERxEUARERlSJSlpXPuVekcugpXyMjIxto+c0lWF5XMbD0F8wwvoZZ/RL3WE9n3LQ3cey/hG8NVOGYpiJu07B326lE3RpCynSct76ArNbQ1NRETU0Nra2tKIpCUVERU6umYPcJdG1rZijop1flwSuO5cmTsopu2UqfYiOqqMgV/ZQn1LgMRbRZEtRFE7QF1EiI6OQ4NqK4RTNaJI7QtJKjCpCTU09p2S6GOmag6b+AVcUZvJmlRiMnWdy/g0XRNoylU8lIzyfPlUeeKw/9e3rXxNv76Hu0gS0GLQ/kxjmY6QBBQIwlUQ+EuUm1kXODbyIO7EcUxkYyjIVVdB3MINEukFCreWN8NY+WHMO47CaOtOxhlqEOSZUgolcjWd7/f1ZRBGIxI+qYCbtHROyJIHQGCQeM7FVXss5czd60ChRBpJAIx+VqWVGeTUaTSKLv0M/EtNeu4vopetx6kcyIzP9r77zD5KbOvn0fSdPrzsz23lx2192xcQHbYNMDhJJQQnpI75X08qaXL+VNJySEkEIgQOjVYNy7123X9vbeZqdXSef7Yx0CCUmA5I2BzH1de400o9H+RtLz09Fz2ge68zzpV9hTaeOymhBvqAxRbbee1G6Sy02QTg/M/GUGyDxtOZsdAySRSAkdR8/AMCzMntWBWzFQ7ENY3VOYhkZytJX4wFISwwsx9ZlzkRKSW91ZworkkqSVBl1FYmIqeaSSx1B0TCWPqeSBLFLRkUoejyWHqpgoIolFhNGIIYSJlAoZ7KRNOwYWssJOZ6CajopqhoqC5DQLTplgqbkTZ18MPdyPquQpNhTK9FJKMjU48gJdG6KiuZw1r3vPCzq+BUMvcEqQ0mTTj+dT8l2DvjnnsnjuLEpsH+dWzie44iDeiYV86dBrmapKcLy1iYbxflqnwqw4dwNlbicZCWnDJG0YTAwOEDuwF218FNU0kHCylAQOw0eDGWSWGSQnXdznM5kyOynKj7GtsY2DlY3UO2ycY3FgGU+zty9C+2AEPW8gT7aOkQJqVJNXezpoWfJT+qc38IDxBp4otWLVDZpGT7Bw8Ci+rIn8q4m+ckqO9Jw0KxeuZE3VGkpUJ2b/IcL3hsmMeshYdnKgdAc/K3kV+wOzQVUIpKdZMHqEkp5hWncf4xV9nVjNPOPF1RhlFfjzKZyxMCIW4+H6FfywYR2fNLdxqf1W1EyUrsEg2ZwF06cjZgWJ1biYtidwubJoWhzE0+JagsjYScSdDKZCxM1a1mbKcaYDRCbCLCjNocY6EGYOhElO95M4niSfL8Jz0aW4V69CaCqoM2mT9n0/ISvuJTZwGjV111GyqJjvD09w52QEi11jDJMam5UPhor4wtA4Wd3k23vStCTyHCwdpbvyCMHiY1QpE4j8MKaZfdrRFNhspTjs1Tgc1djsVRzrtLNz5yg+rwvX6BDpyQVotmrWXD2bnNFOIvMgOWULqNNIUyM71cDIYBM/mzydsGnjk+YmmvUInUYlu4uXMeEPYDFMGmMTzBrZj9Plw1K9EL9pp3Yqharb0MQR8rIRgY28zDGkZ+jJ2YjoEPYotNdaOFzjIOLWUEyD2mwHl9jvwRsd49bRIN6sSkOqnjmxIFZjipwZQ8+bGFJD1zTKzV6u+u4dLyiuCoZe4JSx7ZH1eD8yyFRRHQ3LPk6p9WqOiBqGZ5ViD/ay+8nPcquZQyz20R96Zicbi56nNDrFrPFBqsPjOPSZttkxix10gcvMoApJ1HQw5iql1lvD+lGYnbVgYHLAk+e2siF2K5vJOn1kitYhsyVUjWZJDSdJpHXyjR58XoWW7btIuVXOO30j93E5+yytuPIm5/Zl8Q6nuTudJgFoTpXl7qO8Onc7g1qMTW4HQ6pKMFPM7HyMq9KHmZ/NYmYU8mmNiLgKw3EZ2ew4nV23cyBYzI2LXkmmzI0MWDE0jcrxEc7cvY11u7dRPzqIpawcrbKSTJGDWCKMZ9thjjUt4mNzLmeZkeDtYjtL3HegmRkiudVEDxtku3owLRaO19czVFPO6d4RSpeWYQRNMrlxMvokaRknjSRrzYIyM/SAppvMbk9jj1bjXXQaytHbQLWSnfdBRn61jfSBA9jmzqX0Yx8jV9fG1oe/jL3yd+TDZ7N8zTe5P5/hi13DxHSDRqeNY6ksS4xpPtjeyazxOXRWHuMjLbVMiQCf7tvPOSfqUAw7SdcEHRUnOFETo6GimuUls/E5a7DbK1HVme76mUyGO++8k46ODlpaWrjooovY9ade2h8bJp98AFPveNp0gBJfWZK61gkSpRa+3v5uJtNBPtR8M3OL67jF/xruyduwKoINEYVZj05yRfEPqfUew3jtRqKPjJPaN4EqhhhS7+fnSit2TWW26WG2HsCphthYZuf+co1OnzYz0me6E1fsCNfZd9DqO8HW4XmM9NdSnPMjECiGgSWfR9N1NFNH0fIoFoMmpY0uZ5h3X/+ZFxRT/7KhCyHOBb7LTAOCG6SUX/2rz68BPnZyNQG8Q0p54B/ts2Do/x309f6Mqeu/iXrMjnL2/6NW/S4WZTu/K76YmtZHse7+AK8L13GFZTfVixYyOjrMMWmhv7iCoUApuqph0fPUjPdT2TPC2JSTPlspmoRVwmR5YIiYmicTn/pzL3Ms0kZQ9bIu24zLdNBjG+IPwYfZ6NlFLnk22eE1SE0lvyDIbHeCTxz6Gpoa5MutF3FQm4s7rxPojjI2mEXokkYUlmGQ8Hezw3ecpHoUxTYGQNuYwSt3G5gpFX9CUpQAfxKUp4WVWtyC/RVvAaHS3X8vOx15flGxATTBK41DHFsyn52BKkyhUJbupTi+lXh+H0ljZlagV+23cuUDKcbcRXxk5btoKzG5ykhSEt/CPOf9CGmSHLUR67ERH7QjpWC0oozS5mlaynoQFgfSVc506o2k4ouwFQ3iWnCQjAf2pR8mYyb55fZruOrJfZz1ujV407cjhvchF7+BGGcw+p0f0q21EjnXQcnC23Co5+Bb9A0+c/wE09F25ipjbDEWMCaCXJm/i7fvqcMVmxmtUCIJWyTvXGaj223ji9V+rpy2k9g9ht4XxxCwOaTxWI2VxoVlvL66mEq7ldHRUW699VYikQgbNmzgtNNO48SecR664TBtZ1QSqhxkarAfT6iYkDZNaGwjtp4HmMhZuCz/ecYp4qNrDlBpuxOrGSWFk5hzNctrL6XGv4o/fuo+kmmNV51fRmqPiUzn8Kp/4DFlkF3aEqTIMOVIMRCsZSDUxKS7EoRCVSLOxUOCM0ZMHvbsYGnbzVgUndTBS6l+sB3L5CBaXgdfnnyDSaRe4Vilhc2ePMOpIB8cfjuLk7W0V0vOf9cZLyim/iVDF0KowDFgAzAI7AKuklIeedo2K4GjUsppIcR5wOeklMv/0X4Lhv7fQSY7yu4vnU3w1jx9a97D0kCEkO1r/EK9hJoVWwn0ns+7T5xDk+U49UoYhEBKg4SYwsxME9bms69mAePVpWBXQUqqp3XOnJxmz3iG0kkbs2WMeOAQLgQmJgoKWTVL2DKOT3NxYXwlpdkKPq2G2WlYUF3HsFX+EcPux2ddi0tbylGfD5+c5nypc3rLYqqddvpHOrlv6285mDpGyjWAUPJgqhSlqqjFS+NENRfe/jAWM0/YFSDt0oh4M4z68oz400x4TSJuBaellDo5i1dPn0PICJDpvI/NmS6+vOBqrBicVv8Eh/zHmdJmY7hOI2WbA0Bd/jir9Sep1DdhPZpmzS2CvGrlo8vfgSOg89XM7xjLrKDM3cVsx0ZMYSNfcgZTfUVMPbQbazqNXlxM+euvw0jOJT+SwnNmNd71teRkjo39G5lO9lAS/j353Cg/2PsmVjxxgqvcEapfVYLW8Wt0XyOPJj7IpK2TkiW3MN1XwmioGbd7kGrZzx6xjB/zbjRp8L7udi7sakFDBQTuNVVEdo1iSekkFclHFzrZUayxIC75SFM5K4rc5PdPEN09iprUmbIKHqjQiAUi2Dp343I6uOKKK6ipqZkZ7/zrewhVurnkg4tQ9cTMAGx7fgmj7RianZsDF/Ll8YuRhoX15zRwv5FBmnneGejjLHU7yfAj6HoMDTtFPX7o+gQB00tO9FJt+QY/c61k2Chn0mrhyKJFdCklGKgo+THsqa14E7tojo4S6JuLv8nJ2U2PkYhXMvvAu3HGPKSN3cQD7XQ3xblVD9KR60JoSTQ9xMXhS7hmYiEWIfgOGTJVHn78nhUvKKb+VUNfwYxBn3Ny/XoAKeVX/s72RcAhKWXlP9pvwdD/e9j46+WU/U+MgZaFtM56A2W2q9guFpKZp2DTdO7a9lF2iQmqAw/gVrNUdZfTEa9iZ/EcspqVYD7GxWofDf6lHClxcneJxoRnplLNmp2gJjxEy1gczXKMWcuasMeKObh/P8GEAwVBXLjYp1fTm/fwFmy0FQ3znYrtTBRdzISrlGA+yQXab3FmekjYKoimR+iKjzCRm0nxhDST2TY/nuHFDA4uZ68oYtbkCT6//RdkbC4eWnwdNdk8JdN9pHw6R5od6IrBVNCDwznFMMfo4DgKKu8aeQ3nxFZx2D3Ez6q6ONAxD5fI8s6GP9HYsB2EyRRBtrOS7eZqutUmnDLJRfJ2Lhi5n7IfSkRM4etLruJwXT3XL3mIxooS0kczVHdvp1oeJ2cJYqz8MNufGMO1o4uy2VchNCuqo5PoRU380djDvT33Es/FAXApkveXCYpElh8deCOeIxbeffi35C5fSDCwkYxXZ9JrR9Vm0jRJXAykmngy/SoeD82jOp7nO4Mqtf1J0GZa9gSunI1tThAhJdEHe0lsGSaH5EttDu6rtFDXlyLRE+fctjIubCtjqakysWOIbT07OKYO4xUBEk3LmbOqnnOCPu7++l7yWYNXv6MYV8eNsO/XkEtglrRyb+2r+KCxDHN/BpE3kUtDpLwWList4sP1ZdQ5ZlI4pplj6tjNTD+4F/vYpSB03JZf4eF+bm64kq2TGfaXNzNYchZC5rCltlExtZ3VXcdZcShPoNfHL5afy9IN25kbOA6Dp1Pe24wRyODTV2NO2NEMhYgaZ6vnANusU2ypbOWa8Bze2KeTtal8yEwQdqnc+MZlzCr1PEu0/HP+VUO/HDhXSvmWk+vXAsullM86EIEQ4sPAnD9v/1efXQdcB1BTU7Okr6/vef2QAi9NDm1/F+b1j5JWvVQs/RY+7ZPk1H4eK1tFcPZ2Eps/w/tTAZZGT9BhryBuc2PVcwTNKBscR6k0ZhGllrAtxyHvJD2ZEdI2K6avCllUTrbIBYpA5E3ERBp1PIMymcVpZGlUp5ilTuAWWYa1OTRaq3lgtp1On0pFyuTC6AFWln2VfUnJLWErgr+0lw5aHSwIzmFt7XksKltBrbcWIQT9376F+A1fZdoV4lMr30qfwwNSUGYo1OkqNYaJ23MEwxpH0zUW7evEP9HHkUUTHFwlKYuezlXDVxFWY3w5eD/7xi6gSGS43LeFdU4P3kwdkbSF45YJ9vnSHC1dy4GgB3c6Ql30V3zq5zsoHTT53YKzubVpDe9acjtz/dsB8EXzNPUk8UV1wsolpNJvJKNMM977K7ztnWi65EitQu/qeeA+D1usiKlZnYQdm1jlOEZA04lmffjtM2PIm1JgpGx0K7N53LaaRKSZOe0etrY46C21sb5jhE8PuXDqAlPMNFvfj2Aknsc0TZrXF7Hh0kXk+uOEb+0kP5Xhu81WbmmwsWpcRzkyza5sjgqHwXpbN0omwpKa+TRNVOCZzpNRoEdAOJGkedFtlGTvJuG2kCguIWV3MmoIxpJebtx9NRndxpuX3MKsohE8qkATEilNYObVNl1FqP012JPVpHw7SfBTNrszPOp3cYJSYqG3kbfPoTi6m9c8cANr9kYJxSDvhnsXLWdrUwtvnP8b3FqGsqPXEjBXIFfX88W+O9g+fQ9WNcKS2GLOnjyTxblKLELBFDPpt+1BlS8HDeJdcfKG5NzWMn587ZJnC5d/yr9q6FcA5/yVoS+TUv5NmxshxDrgh8BqKeXUP9pvoYT+38PE5Ea6rv8A3q1Z4q/8Ns3KRgLWn/BT+6U0LttE6Mi1XDn4CtL/dE8mKDk0YeC1WNDyOhZhEKws5ZgLYi4wvV7QZlIzSsagOG2wTkyjTxzj0cqFTLu81CRNru1OcXa8n8FXfJHpnJ0fjrpBczIpwkgyoDjw23xk81HS+oyygM3Lh3a30nTvFuLuKhLLF7Kg+H6KzCEOGGv4kbmeR8xqJApWDCrNFCXWMDWGQXW0CUUBLRCmutZByO8hdFhDSQu+V7SLO8NzKBZJ2vyPsy+0HV37y3jpdsPK5YmPcndtEye8KrZ0D6+759dc/dgRnmycx7eXnM4rWwOsbsmTNseYmBjAOGxg5ALoai9Zx04mfIK4UDGnBHpEkBOCjE2QcStIuwRFwYLg1aEU1TadOyf87Iu5AQWJgpCgSEnaXstoxesxVRfn9rTzuRONxJQUDmkjqaT5evUtDNrHSRspMBSKU1XYVQ+lCxdT71/C0sNOmvZHuLnWwvfm2Fkc1rmye4TjqQPopmRzvg7hkyyqm2aZNcXcHjfBqdmohoOcY5xo5ZOES3ehJ7NEsk6OeGZzW/vZ5HWNq1dsZ0O1TsBiBSEQKDND3mY18k9W4R9oIqoluCP0O+7z7SGhKCBBs5/LeOhyVFPy3t//gvO2PUFmgUm2VdJfFeKXQ9fSEOrkVU33oqZKqN3/Tno9Kl92PMGYdSdCyaMma2iLrOXa6rOZvbCSoGkS++MJZM5AKgJhSKYxeQKdxxSdOUvK+PJlC15QPP1HUi5CiPnAHcB5Uspj/0xUwdD/ezCMFE988yzKfxGh94zzWRBYSYX9TTzI6ViWjOJM1XDT8RVId5pk2IsTwXyCzMpVMWwb5Deld6KlXMzNzOfyVReguuPcf//94PHyUNsr6FIUpDIzMl/ZdJ5QxMBhjzDih0HHX6anK0kmmdd/lNnTvWwqv5f3VWRwSRsN279ATjHoK3mCpLebnH2avCOCUNNYBTgUBasicTzsJHBXFutcL8Vtk/jEFCm7QtRjo3Qii5CSuLmCR9NLeWTUzu5ANWPOIABuspRaFMp0N/OjErepYhWw2KVQqqlsFuN8RtoIKVEWu3ayvXwLWTWLMIGZLAaKVEl5VpD0XYGpBajv38Xnf/4bovYxvnWpQtLxt70xhRTYTAWLqaKZVhTTjiZtWHQrmm5HkbanKpMBLMLkgoYOarwRHhlsZG+0HBA4MinCwVaO116AOx/l84cinDFZwrBvitKYnzAT3KT+iKFKC53WNJa+M4hkX4FbDJMt2Yfqa0fRkkhUGvOv4CODF3GkKMjn2+z4U3GuOXCApd4ssvlWNFt4RowJ2Xg5yrgH72AdFtZRni/HBNrdeR7wSh6YyIFh8PHgKNectRT7ghmTzB4/zsSOzWzv6GeWuZwi6eTuoif4VfHd+GSCFtPL/Lnv4IaJUrr95czu7+bTP/su6YCLzLkVBGbZ2Nazhtv74Y1tN9NafAT3yDKsRy7l08Hf0xfsRJoWiscbeLV1DZdd9GqKKx2gG8QeGyG+dQrDr3KjfYrNYxMsQbBSUVmEwG4ajM4OseANF76gePpXDV1jplL0LGCImUrRq6WUh5+2TQ3wGPA6KeXW5yKqYOj/XWy54xz8n+kjPLeU+ob/wWd5J1OqQXt1Ne6aPgb2vI69/pvQMqu5buxK7NLKdutRdshJAulqyhZEMOaVsffwUVIjXcSdLiLeYry6TnEqR2naile34JEaLmnHY7jx6W7itiKO+ouoTqv403vY4zwA0XIWL7kbpzMGhoJ76DRKe67AkisiWdbBeP0WkmaSeNIgrEaZsE7Qcl+AVZNd+OdmsdmzxFwqnZVeHrf6iQtYY4nRvNOgUqRQ1RwZYz4p36vYlE7wp6koxx3zGJVF5NEAE6t1nBKRZHbWwWW5auZbXQwbJu9Rk9iVMOss/UTlFAO2DlIOnZRNYDXslCa8BMxqOivaOFTdjKkIXrnpES7b+Ag9reczO1iJU1rpTGiEcxZUqSEAlRwuNYxDiZMWkuOiiKPWIhQ3rO0+QdNAO57EAMLI0NlYT+aaYRo9x7hv8nIeOrCM1PxikiVe6iIJfrwnh1+XDKkj1Bg1jIkoD1sPMGyZZodpEomvRhcWzhg6QI+vgj5vGUKahJQomeJN+IIHsJmwZuAc+nylPNyyHI+u86NdOSrTGU6I+0j072TU9QE8iSTVHTfS5S1lrLSO8sVLKSlpJNeT5EvZJAbwXZw06Hny8WGOax3s9RzjuD/BOfoFrEososc6zC+D99OgHebqSDex0AX8Lr2c389eTNZq5aItj7K+fRdK6asoqvLhM+8mlH2cjDfPSItEt0JTl0HpSAZdZFElaAhUaT5jCH1dhgjnPkpOtuBS78ev/Qwhcs8aD/tLzmfhO3/7gmLp39Fs8XzgO8w0W7xRSvklIcTbAaSUPxZC3ABcBvw5Ka7/vX/4ZwqG/t9F1+FvELn+12hhHf/qH+NXb8aj3cqNgXOpn7cL3/638XPLQXy6ByFUNEPFKa14DTc+wz3zqrvxGR58hguLfPbZe3SRJ21JkNFi5C0JpMPE6vWjDh+Cx2+n+B3v45B3F4q6iXjsUi644Mv0t+9j262/JRQpYa7/NBRVxXN6Na7TKxk60gU3vIXq0GFUq2SPxc1PAg661AqaJxYzK+vHNz7Iwt37cCZSTLYahJe5WJ+ZwCXjZM0GHrXP5ofFYWaNtZDJVTGRgOm0my5vJVIoqMLgck+KtyYqyEn4rEwRUSdYpfThi7ZgzfkBMIVBRkuiq2lKkgI8IR5q9fBknQV7NsNrH76PFckafhmowW9kiNtNYppGStfI5QQZ0yCvZLCIDOeoezhP2UGJiNBh1tJv1DJMCWo4yQU7n8CbTTD5VhhYEOKb2U8xYivl8u5JPnrCzgRRptQYLUY100oXfyju5hG9jNiYj4TmZ/noYZrdh/GccYLW4zmMbS62WBezsWoJE04/JSLCWttx7Ji4lTD16gY+s6SavGLy7sMdXD5Wy7gW5lfFd9NdPkpDaDapRAnHBtz0j/ox8z5AYJEm681+HPZjjPkG6CoaIKVmOX/6dN48fgkWaWFCC2PTLQSVHhz0sj/h5IamOTw2r43i+DRX79jKnPx8jioGVerdXKLdi0ek2VTWiN4UxZK1MOtQE0P5aY5bEzQG5rCouhlNtYCiYRomqe4+pg9LhO0yTGFhl7mVfbZpLKpKwlCYXZJjVfn9ZDMQ7b2Q5FQxpUsXseTac19QLBU6FhU45cRi7ez91HsofWic8UvfwRzppNT2If4oNuBbcQjfyGrKjv7tcKJJJUeaNGmRJSPyZK1TDCppfNF6bJkASqCPYNsBFH8PSXkMqebw+ZdQWnIexSXnYrOWMv7VrxG+6SaK3/9+chcHOXr0Y2jqOWzcWEJ9XR1XXHghajpN77bNdDzwKJXWxVQWleBUbsOtPowidMITRXRH/ExNFGFLJHFmkjizJtrJfi1TfhcPr6jiUJ1KxhrHo6V55ZiHS3JduJRR8mY5x9OX8ICsJ+YeQcu7sUbmMqRY6dUMejUTjyr4Eg7qUbiBLDvECKdZenASIDk9jOpzsqU2Tl88jql7ELqXUKoWaStlotFOotxLydQklQd6OJKvQDDTm9atQCNRzhG7WKvsYLZyBEVIktJGDCdOsvjEzBAAphT0yDKG9SBbXIv5yfKrUBWDjw1u5JzDZ9Ar9+HW3JSYczjh7mJQv5cbo2fRbaumebqf1Zm7Caw5RmuFQUYIhpMqtvES/Ls0yvfH2N28kJHZNSSljcfyzYSFoNQ6wGu0Km5Z1MCUTeGte8fZEDapwE2vfZKfBP/Efu/TvEK3oaWCGJYM0jGTnjHzXqojy3j/9Fm0GT5GjCiDZj/lNoMGmSKfrWJbSTVfnO9mwiao6+sncCLLhKlwufog12n34BMpntAW0jsnS1VgAstEK6WH3sKvS3s5/ewNnD1r7lMSDFOy+cQkd+0eoOZQhMtNCycw+BRpUnqC9V1b+WPTGpozg3zc/wPys15Dz941GAbM3eCjdUUNoVDoBcVSwdALnHKkNHjkB2dT9b/DjJzZQLP3owRsr6VXlNLbAm6HTsVPzkBG+pG5BDKbQOYSjJaG2LpyJYppsmz7DoqmIwjFgrDpKI4MUjNAA6kJVJsbh6cam7sSabExlTXp6x1jMCuosBi0LbMysHor9mEXxb8OccLpYdeC+fgjEc54YhP2bBabL09gbhJvbRqBQso4i+noKga7diGNI1jtBtNaOVHdCYogY9UY9UsGKupxSgdWnpnHloqkjSnOMbbiFV3o0s8BYyUPq3VkcOAzSylJKShT0yRjCXqdAeaXz2Wl6mQzeW5klPm2XtwnH92lhKi0MykdxKUNmxHAKd34UMh54UCtSXdFBaWjE6j9MX6c2Earsh0nhxBCkhdl6EYNebOCEWcTe91xjvsUyrOTzEvux5bKoWXh1oYL+Vntq5kf7+D65NcwS2O4u12E2t+BzdlGMnUPX8DPFud8SlJh3jl2J1dWPY7doz/12w1gt93GRquT+Kgbr7KOtD1IaDLK8ic3cdRXzT0tS9jtb8XERr0N0otLGXCrXH44CcNJXoeNUlQ69ClOjPyejHmA7honY7P8+IsrmV95Bt3Hi2lqF5xv+jAwGBZhFAQN5mNoPXcy2BPia5e8hY0rV2NNZOFgBHcswVvVB3mLdi8ekSBmLGOodC6jsx5FCpPA8ctQR9fguXIedS1lJ4+95MhIjDv2DnHXgWEs8Rz/I5zMkiq3k+MXljzXriqmzf8QH7mnHoee5RuHf8OQdT1DlS0IWx+mP0oyn2VObR1XvvENLyiWCoZe4EXBnkeuxPKpQ+QqBaXNP8CnfR+b9ji/r1tGZU0H2bQDLQO67mQyFySVcSB1OzJjoeTEBKFUApucwkoedIFV+LEqQciDnplGT0dAN0FXyGet6DkN8mA1DbJ+hfSH4mAoaL8qQZcgXAZToSKOBBbjFAkuV++izpgiL22M9ZWSPpClY/V7qS1qJZDTGFJG2aqeIKpmmTKdTCoODrfWMB6qpXS8B//BNMO6izr7GBfWPIKXLJmsh1g4RNgMsSI/xlr24VcOYkg7O5nHFrGAmHAhpECYkkA0TmvEjrvkNErVIkaR/EDNMmTJU02KkJHAa8ZACyNPJnCFBE86izeVxe5Tibvgpws2MOIs4ZzJzby94zdUHZ+A0Tx6UkV4StjZspqvr7mAQa+f83oneH+PA+/wYcaO/oEvXvN69sydx0UnhvlsXyfp7B5G5h4hVxfFN3g6c44lcKiP0G2WE43Ziahh3LPzOIoUpnKC4TELReEQoVSMKt8EFnz8nguZwo/Hvpt6cZCRqXfhPDjI/MEtKKbB7U1ruLd+JVGXF/+iYoaDVt7Wmaaqt4uAkmehWYsFjW4xyKjsxB8JE3E14LNWM0cGsfCXiS+MzCTTgz9iR6yIh+auZPv6FRhuK2pvnJoTvbzDvIvXiI3YrDqdOSc3NJTSXJ6h0ZEnlajGOvl+xkeLuafUgqXJi/BaGIikGYqmSeYNhCo4PwYfPpbDNCVfIc3OWhtqo0pOM0kaLkwxM8Jl/fgQzWMDlMWnkcCox8KgcxpPTuV37/vaC4qjgqEXeFEw2HMzfR+7Af+xMbj0k5RmxglZv8CvlVdib+3GzPsx7Wks1iQuLY5F+9sKJSnBMCxksy6yGRe5nINc3k4u6ySft2G1JfF6J/H7RrFYcxiGSnS6Ao93HM2SxZQKqmI+tbPSiBO128JdyXOwoLMwEce7vRfX5AAPnXsVI34oTkRpsrexQK9DIPgTCbY7d9DY6CQUqaddlvNIsxtDFcwbiDB4IkLcsLLCt5PXLf0dVlVHz2jcPXQljxev5XW97bx18j786k5MBPvFXLbKpUyJoqd+p1VqhMwilustBKVGDwZ/IMd9AYU2q4XPj8KUPcYNRb8moGucnUyxwOyglmEA+kU53y+9mlsbziGrWmk80cM37v4xFs3FD9edz566RuomRjjv8B6qUsewORsZKd3ANxeUEbEqvLcjxoVDWbYEEzxiN/nQWAil+k4ijffQO1JPYKdKpX2aevsoxeIvbdajug2vliWlW9g8XkfYWUaPfz4KBhZziiNGLbvM2SSYmRavLJvhov7drO7fRHE8TE/Ix09WtNCx5GqipSGu6s2x+kiMjXqGtSLHMksQefIhSD05SJopTTKRLrZlhtlstzAkXRwN1JJr8mHUe7Bnsly8Zzcb9AdZK3biFhn2OW3cVeLBHVKYbzcRQDhdwpQjiFNJYCfHlFnMhFHOmFFOv1lKWK1Es1TwxgNZzp4wOIzO/ysxsbZEsVgH0IDjk624JyIszYzgjk+hAFklhUx2Uz7Yw5z+FFWTsHtxPa+/5b4XFEcFQy/woiCTGWbTZ6+j+q4uRq9cTFPmDZTarmKfaOVgTYCh3iU4HVF0IJt1UVNzgOqadtLxYsgXISzlCKtGzoyi62EsShSbNYXVmkGIZ7mOdQua4UfJa+TcI3i7L6Mz4WI4MMVptv0s6DuKL5UjadV4NH0J+6lHNdKcsWkTHXMWMFbiRbdaccfjRBQfG9WFXGc6OEuxkzElRzMGgyZUNPnxLQ7wvfwoWxUbxZMjNB4b5WCiBLdN4U1z/0Br6AmEgCFzDt82Psyoxc8HDhzlusk78WqPoZDnOLMZDy9lcipPTMmQ8LiJ+wJUatXMMWopk27imNxFnm2MsMq+nbO1LSzUewA44qzloaLT2WWZh562UjsxjALsamjhSEUdmmGwcOA48we7sJh/aecugSPldWxpmocrm+HsIzspTkSfcSiFFORRqKnaT2PjfoanyjjSswiHaSWVcKKZEpeaJiQiVIgpfCLBJEXspw2fnCamm2wxW5nU7cxTOtmgtXOWdoR8vpT7Ix9H6hZCk3txh5+kdnSAaZeFT77lDRydcyYbRvJ8+GCaH8kMo5icJTXOFBYUBO3NNjbnJ9nUmyJ68iZR6Zlgen4lYXcxa9Pb+Njw95k7NoI9bxL2W+iudRL1PbNSPYmTmPQhEgLfeApXNIUeAlGsIxwzBQBLspSKA+/Cnqihq2Q70aJ78CeHSCds9NJKVzqEjQCq0EBPERrrp+1oHyWTEQSQs9iJeeqIe+qxLl/I+i+87gXFUcHQC7xoeOzmDZR9eZDoWhtVvu/hsHweTe3kNyVrqaqPcOTAbLJZJ7VlvaSzHozuNWg5PxXaYapt+5GWETZSwyPa6dQ2tTBPGyXctQ+HQ2fVqjaKj+0lfvAgSnMjui2Lbo8QdY4xPVlLc/+ZlLpvI5jbhj2nE3da2a+to73njVhSSVqO/Yxty+aRcjmRQGXGxBdq48ZEgI5cntk5lXPSFuq8Jo26TtDiJqkk0OcIOvY+STw8yVTrYp6cs4RpBG2KxtCBSaYSOUrsY7y57WZcjhhashT7iTdRGivGAOKuKIHc7wkoD2IXWUb1Zk70LyE25sSRnkR3qKQbq5jtbcML5CkjRYacupshpZu9+BlVvAhVx2vY0ITxjGOuGAYJzcrjc5bSX1KOK5tmWc8RZo0NkFMUNs1aRFdpNXXhSS47dJisClGrijulsCjnJSkMturTWK0TxL1TNBf3srjmOOFoKdsOnos0baimjsvMzMxM9LRJLIrzI1wiH6LSOlN5mc2qhCedTIWdxJM2NFsOr9XHTv+7iStlVBu/JZEcw9mbp3l6kFvOuYifX3IV86YS/O8+Saeh80XShDFASAyp4tBStAU7aA0dYaK0hj9pl2Enwydiv+Cqo/dgz+YYdXi4u8zBAa+DGoePBZZx8sLCH7iSsfhyXjk8yNoj+5EnupCjI6jp1MzwzAJ6a2voXngB54l5mEKnq+Rm8lVbMXQv0/0NjCXqySguND1P1cAgtb19FE9OkPG7ifnriTjnEnHMRpg6/lg3/ugJKltKmPWjb76gGCoYeoEXDYc2v5v0p/di1aZwr/ksRemjBCzf46fKaxilFCEVpOJmzroVDMZCHNw/BqNZGnUoMmfypA4lSpW1naqSaapfMYdc8xnc8/AOeob6CBluVutzKQ6G2FxygO/lb6DNqfLefJbm/m6suiTs8XIwdwFTI1cwkhN4c4O07P8+llyGLaevZaq4FF0Y5OLN3KX5yCrw2spiXruylqoqHXuqi+zgITo3TmKPLcap+f4tx0aQwqU+gEe7E1WESZtNHJUXMa2mSchRYgKm8RGRPkzxtDHZJRhYmTBtxKUVRQoCaDQGknSURdmTHeVTNz9J4wh8fcObeOSsNeg+K0WZFIYQxKx2zuod5LS+HgYdeaazEl+6kXcSZBCDW7WjBNTxZ7S5Lis7RlPzDqZilWzpuZC0cJBXVLKKBWGxUptNUjE8yKxEAnd4Asf0RY8grAAAKFFJREFUUco809iLcriDGay2mVJvNqaRHLMSH/ewK/AWxoqWUj3wKE1ddyBO9nh64LQz+MZrr6MpHOa7hxT8hslD9sMcjfWyINNNpSdOSekkn2n+FDsdszkzsosvdX+f+ngf3aKUjXIFt8/2kgtUcoX2GE2iiwP6Uo5PvJX3bFhOS8nMmCrpRJypgT6mBvuZ7OnmxPEe7taruNA6nwuFlRPmCI9nfoon68Z01ZL0FiFMk7LRUcrHRyhSEhjVEr05BU3TCOtfZkcSKYE2BtqEQJuyUBRYRfP1P3th10nB0Au8WJgYe4iDH/oh5bs7ib5zHTXD51Jhv5bHWMnj5iqGyuwcmJ7HZFxHEbCktogz55Ry1twSKqwWBjunGTgwxGDHFGYmT70tTKNtArcaJ6r0Mq324BRxrGoYN3GKTQPl5DUeq6iFZe9neFeQx3cr6MKJau5m+c7foZmQWPkmqvwLyEmDe20HmBYRorKatzR1Mje9GxE+jsjGnvotpsVFr2xgKN7ILMs4PpFBomCiMS1LOGJdwf1VCwnbLCyaNlg3nKfLzBPz9DG3fCcKAo0qpoTJLnMJUQI0iix1w0OM5/YyLBSSYqYHrEPm8KoqHiWESxbhylrwmU480o5b2lFQyCPZg85PyHIcEwuwBFgmoFXECU0NUTQVocMd4OcLFnF0toecpvA/7WkCyQjDB37OVlsrgZrVvFs46NLH+Grdd1jis3Iop5PKWWmxL6DaewbpaAQH21hQ+wjTsWL2dJyNoroxc3kseh71pBknrHa2NM2nJ1SOktFxRhPkoiYt+kNcmHmcCyadVFoPYVVymBKm83X05JaQypUwK3s7pj/BcKWNnYnlfH3JBwjE4vy/7WM0WmpJ6lmkGuWJKpOvzanHRPDRzu1cO7qVIWnhsKxgRHp4UnVSU7+H9bXbyOhW7utcxYnJOmw2A9PMIo0spp4CmUdIHUXREVkP8yZreZ1/Lj6bl7Hxreyx9jFaWoJUFDzRFP5pFUu2krSzDl1zgJIg6ZxGtUZZoIYp2XEnZqnAceVa9BJJWo4STXYDYeACzjrzey8ohgqGXuBFg64neOibV1H/yxNMXemkKvNtXNYPElajXKB/EodawbrmIs6vF6wszuPNT0B8FGLDEB+B2DByegjiowjzb6dPy5guosJJXDhJKi60ojDYnYzsWYPeFWOgailZ22wMNYMqt7D2yfsQqp0Ts1+FpyxG0B5ln1hNGcUMW47Sr06ySK9isW6QNqeYMnKM5r1sFuXca/OQFYKWvEpDXjDbnGaubZgap4bTrEKaxeREjpsbw9xQX4/bzHBt9CgPd5QQ02O8b+HPqfQOoCprqKpZy6/29TA2UER5JIwEFKBSTtPgm0X5hrX4izWU/GESJ35LfHoPx8wybjr6ZlLJCi4rGmdFrojSZBE6Bse0KR7ExmO6YBqwAK9AsN7UWK3YcSLIC0goku25CNvD3WwpquWtiovXKA4OM8xtvr2UOqMM5fMopqAxW0PGUFHIgzQRuk6wpJ/ZrdtJpYIcbD8TXbc943yYEhQBk5qbJ8pbmSwtRjpURMZAZAwckSzu5Bjrov0szR9hPgdpFcdRhYkuFKJ+lSmvFZEXjCdrefOcr0Je8s0/3E+ZFuJraxfyZE2IJeEknz8YoyztnBm/5c/Xg6eP0dafk/X24xleSmj/K1ETPNUs9qm/v1oXgQbs815DHp2NlsMMWmNYcwJbKoSWq8EuXBiGRCIwlDiRwBE0I0bb8DBtVhvZ7dvRSkqo/eUvoLSUnTt3smXLFtLpNLNm1XH66Suprp71gmKoYOgFXlRsvvM8vJ8bIT8vhX/25/CntxC0/JpDZh211hhuffqpx+0/IxUL0laCrhehZ/wYBFHKqjHyWQbu/SN3bahmm91LQ2Qei6Kl2NueZGCygmQygCuuoeoeVKMeTXcz5Qvj1rs574FbwObGVztFyfwxHlPP4qPpK8lg5f3lk5xRXMyu4Ql64v3MoZqVmWYUBBEh2SHzTLg1FreUU1viwh3PofbF0PtiIMFa68W5uARnbRYl3E7HSBcfzjex21bN6eHdtB1t57fJNbyq9iEatDBjo03oug2X205/mYd7Q4upyI6woaMdmX76cZBkHBpTwsN41EUEB5OzyohUBDAVhdqEwZV9KV45nMduWugJJOmrreCB3mn2heMkhUAzDZaPH2NxbIQ9/moO+8pI2jx8LWuw3FbEbeT4Hhn+ejplDRMrJhbyWMw8FvI49DxtwSNcsOxu4jEfWx49EzWpkNJsdPsqsKkGAVuGSjWGAI4aZezTyzFQn3l+rQpOd5o1lVs51/8odalJfBMa/kkoMmfy70mhsN9dw3tavkZU9WI1ciQtTj7V8xMu6H2ALbmlbNWWMmdyktJcBm3JBI6Wg4iMDevWpbgGmrCqDiyqC4vFiWpzo1jcCM3xrNfpkBLmCbUDM+vHJyqoa6ilormIfFZnzwN95HI6CXcXOcc4znQlS/w+avf+ntzhgwDoqkrvihUcrqokAzTV17P2rLOoqqr6V8KnYOgFXlwc3/tFRq7fRWDqGPp151DafQZR69cRPh8HIk7GRYDaukZOXzwfl1ZMqlMlcTCHzJhoIQeuZWU4l5Qyddst3PvHr3GiSqNqQrIk4sNmDRN+Uw7TCeKeBrr0VUy7qnAm69BVyYOLLFx5cAfLHv41wlFEybVr2Nq2jvc9ksBAZVapmx9es4SmEjcw05nkkUceYcuWLWTtZUQjday1OVgmLGiZk5WPqgBDohbZcC4uxbWoBC30tyZhSslNQ5N89Vg/VeODnD7UAfE0EigKDFJedpxZE1Ms0oLcU7KaT7nOYNRSilXP40/FKUrGCSYiBMYj+HJJPMpfmnUaCKJONymrjUAyhhbP4Pa4OSc9i1I9yDAm95CjQzdJmZJeJUdCmzHVuYkBPugOMJdyjqhP0D1vE10ZBw/G3NhyXs4Ln43V9JCQBkkkKQRTeoysTJFSrCSFhepQN29dfCPhTBHf3P1uIln/U9pURVDrMmk1uwkaYTJSZcp04cBCrXRhs40QnX8ri90prMJkIF/DQ+aFtOfO4DVPJinLTJJs2sfc/A6WpI6iW2xc2/ZVVGnwwe6fcnvxudxVcTb2TIrzDmyh0drP3PonsPtTxE4sZPTQGyBrIehop6j8cbTqTswQ5PM+0pNNZCYWkZ1sIJvXwRHGJlTUvIMJJIHSIqpmBambXUHQX8LW27rp3R8mb4kR8XZS2drMJevWsfeeIU7sGceZHKFN2UfqvKXsOHaMlGlSNjZGa/tBQpEIjnnzcK44Dc/69ThaW19Q/BQMvcCLikhkHzs+8VVqHmtn4v0eanu/we/I0znHR1uFl67hGEbHNBdjZR4qKALHXD9acRp9rIPskcNMbd2EMjn91FRvsshL+nw/kdO6MBIejJ73ojSsomvLKDKuc7xc4/E2+Ma9D1K19Q4UbxkV3/ky0bnzee9v97G3PwLAaQ0BfvGGZTisM2Y3lcjy7YePcWDPDpZqg9gD5bz7La/H5bChj6XIHI9gRDI4WkNY67wI5W9HPPwzY2Nj7N27l/0HDpDNZIg6XEzXNrGoag53bNrHlbNvpLmoh+RYBSuPdeMjzZ+K12EqFkrKW0lVXsi3NsU4MZ7g0sWVXDy/hHtHhtg1NkpRKs7cTApnIko+Ff/LP5VQKjwsMpqoygfIC0lnERzWFEa6BylOG1xQVIVPg5E5vyFW/QhPDFfyJyNK0PDy+fuchDonsF7xFkaTVrx5P35bycy48WLmScQ+qwhbo5+k+wjth65DUQOoxd9jPBlgRWOQEreN7IkIic1DdHV1sVnrIKFkEJYczY07KS3pwTBU+qfK2R51cliNcHlskivTOgfsSzgRfjPqlI1dLRYeaHXRmBlk9fRexmxBHi5aQVt3B2uP7SCYT2BdPElV1RGyOQu/mRZkhOB1oprkwDlM97ZimqBYdYSpIfMWTKGTdg6Rdg0iFQOr4qSmtpqq2nLGx8cZGRlhenr6qcOp6DaEVBlTYM6CJbz5gpXY7XYmvvc9jt66mY75r0eXTtKOEUKtJus2nEF1WRnpfftIbttOcvs2MgcPEXzbdZS8730vKH4Khl7gRYWUBg/89Hzq/l8/sYt0/N7PEY8E+ZpPZUVScC5WvAjGZI4jkW4cXY8wd7CdP2dG0w4rI0EX4YAXqmehl5XhrH6SUMkx+uKz2Nr7dur7vPhTkoGgxpa5Gi5zhK8+sBXb3vtQimqoveXHbEo7+Mht7Rim5CuXziOnm3z4tgOcVh/kh69dxB92D/L9R0+Qyhtce1otG0rTPPLAvZSXl3PNNdfgcrn+6W/N5XIcPnyYPXv2MDg4iKqqzJ07l8WLF3PIVcSnTgwxldd5fVmQ3i39lMmbuaDhYaaTxSyo/TjNVgtULOLeXsHHbm9HUwXfumIB/go3H+4c4Ggyw/khH//TXEmFfWYWp3w+z9033cjB7h5Mq42kI4mpm9TlKmjLV9NolqGiMGKN4DMd2EyFkQXfY1JJ8FB3KxtLN9Jknc2Pz/kRIbeHbR/5ALuHe1E1jfPe9UHqF68k+lAfqZ2jIMRMohwQVgV91hg9FZ9H1Vwsmvcr6HSS2DKMPpEGp0ZHLEZ23m6m1RN0d89MaqYGsjSHFtDZMYBBFoCElgBbkgvHTWZpjexJLKY/rxCwQrZokvbyNEHFySsHyynS4+yp/yNq+S6czjgjqSpuGMlTNTmfhVMrcSslGAkNzJkryFCy5CzTZJ0T6LYoEpPq6mrWr19PbW3tM85fOpXlzh9vZrQ3Qs4WJueLEstncYn8U9t4pESbniZRVEReKpQqLcixIHa3hdVXNNP8itKZ5pwnyU9GkLqOtawwlkuBlwm7H7wc+ekpLMXDyFedR/GJywAwTYOhSC+P5mPc5Qkw4ZzpPenRU1QZUxTZYviccYqNLNbquYzbdZYX/5pidZhHh96F89BCQnGTqF8hP19jVo1Kg4Cib9yE0fEEWtksKm7/Jd/cOswvtvQyr9LH969aRF1oxpz/uHeQD/3hAFZVIaubrJtdzCcvmEvTyaZtHR0d3Hbbbfj9fl772tfi9/uf9feNjIywZ88eDh48SDabJRQKsXjxYhYsWPCMG0Ekr3P90QHumIoiUjqVA2la9N28qumX2NUsWN7C1qlV3Hmgj3llFt62OsiD6Tibk0lCSo6LrVmaZApdT6MbaQwjjWFkMM0MmfAI2eQ00mVnBwke11O0mH7OM6qomlpCbWwemCoPWg+Q9kZIlGvcnv8Djan5rD10LVas2BxR4uM7CXljtG3dijsYoup/v499zhxywwnCv+lAn0zjWBBCcVjIdkdJpDoYXPoNkALnVCtWWwC1xEf/WDtW/zEULYfbNRuf72oefCLD9MggadXNtZdexMCT/RyZ2IFNtzFNCkOYqFIQTKtYEsXkzCYMS4yI/yimmiMkbNTX7cdWtZtcxsPo/tcQH56HZsx0MpKY6JYEuiWOrcigtMGHoWTo6e4hl89hywVxxGooCZXSsrqCOaeVY3dbME2TLY/s5cDd44i8HUt5nLIzG/jA/QOc3hzie5e3MDLQz9abf82gqmBo2jPOv8/rRyYcGBE7FRXlrL9yKUG7g8S2YZK7x3CvqsB3dt0Lip2CoRd40dF/7AaOf/xRyjv2M/U5P4Gdn8Q+uA3TFcEyuw5by1zsLS1M2P3ccfgwv9m/l2i0DPSZ3DY2hZbaY1xX/XNyoy0MH30jlrAFV9DGKxYHKdOj5PpGyXYeJde9H2PkANbGRYif/YD33n6IQ0Mx3riqjo+fNwfbyVzyoaEoX7znCDt6ZirhZpV6uOOdK3HZnhmsvb29/Pa3v8Vms3HttddSXFwMQDab5dChQ+zZs4fh4WE0TaOlpYUlS5ZQU1PzjFIagG6Y/HJrL99++Bg5nwXb4hBTSC4vKaJ+qBdn7MvMCpx4QcdXSpBSwzRUzKyBKWxsz6jcldKp1jSuK6nAo3iwKg3oyjoe37YFMiBdkgvWXYAzbWPzb58gn69EUYsQClRUaPh2/pHQ0E5qPvcJfK+8EDOrM/3HE6QPTGBr9hN4zWyQMH18P93hr5GxDKKb00hpPKtOIawYeReptEIub8ORLaIkW0EmrxJL64yYEDUt5PN28nkbWqoMb3QOisjgq95E0bzHsDoijA7MJ7r7zZimBloSaUuRdcXIazFyevbvHyhTwZ4twZmpQM26QZhowQx5mYKpIMKi84pXVVLUWsulP9xKZZGD3711GV3t+9h4zz0kLBbKbDY2vPrVlJWVMTo6yvDwMMPDw4yMjBCJRJ76V17TQQgv5aVlzFk9n+oFjS/o3BYMvcCLjnR6gI1f/iD1fzhE+Lo8OcsXWPeeq5/63JSS7eFRvnHofg7EdALMIZhzUhrVKQ9nafPfhdd5nMkDV5CJVuFUYI5dpVLJY04cJT+yH330AOSSoGg4l62j/X0f5fo7D6MI+MYVCzindWYUvfFYhm8+1Mkf9gzid1j44Nmz8do0PviHAyyq9vPLNy3D/VemPjo6ys0334xpmpx33nn09vZy8OBB8vk8JSUlLFmyhPnz5+NwPHsLivbBCJ+44yCHhmKsm13MFy5uo9hn5zt9Y/xv/xheTeX9pX6m9txEqTdFj9PNMdNGkWrjIq+HRqcHi8WJ1erGanVhs3mwWl1omhNVtSOE9akbSPe+Xdz9ra9gLy2nr9XLveI+fNLHt1Z+i4XNC/nk5k/yQM8DXF10Nf5hH6NjYwg9jzub4JJr34i3qJETe8Y5sXuM2GQGIU2KwkdpmOVg/seuwe6xk9w5SuTuLoRTxfYqg5h9HxMTD5FIHAHTzdSxFcxecBl1rSWkBgdJDvaTGh9Gz09jWONkbBFi1nGwJrFYMqjqs98ATF3ByNrR80U4/EOkUh6OdZ5OSjZzwuxEcUzx8cTbcUVVtGIHtnXltMdPsG3bNjKZDDabjWw2i9Vqpa6ujoqKCkzTJJ1OE5/IEe/XyI3ZwVApnWPnwrcuJSEll/xgC9mcwdfP8nFg+2YisRiBcJgzli1jwTXX/M3N2swapPaNMbm5l9HwOONKnD4zSsyaIE+G008/nbPOOusFxU7B0Au8KHn4Nxso+9IE2RUpWHE+njkfYnA8QSScJh9NE8xASUYSzP3lGjW0JMer/0T/4DzSE7OxoVOcHUGNHEWd7qZiohOrkSNtsTM8eyHODWfTfOEGfrhjhN/u7GdxjZ/vXbWIqiInmbzBzzf38IONJ8gbJm9YWce7z2zG55gZ5+Pe9hHe+7t9LKjycdObluGxP3P8j3A4zM0338z09DQWi4W2tjYWL15MVVXV3wT4n0lkdb75YCe/2tZLyG3jcxe1cl5b2TO2P5pI86HOAfbGUizzuTiUSCMlfLS+jLdWFaP9g4rXv8fg0UPc8bUvYHW6cF56Ov9v8H/RDI2gEmRQGWS9ZT2nW1cxeLid2HQYtShEVoKqqjQ0NNDW1kZRURGpKZORI3F6tg6QNuwIaVDTplO+cADFsodobBuGlgQEHk8bWu48tv+mhtoaP7McBpYxA81QMDEZVaIMKJOM2+NYHAa2zuM4x0fpq6snEXBQVeWl19NBp7mDplQtStEUkigtGTul0TwD1iaOjDQyXRUlMqHQEdrBR+Z8gmuXXUl0/whbH9jEgcwJskKnsaKOdeevp6qqioGBATZv3kxnZycWi4UlS5awYsUKfL6ZHr96ziAxncVfOnONXPXTbaTGeljvnyIZixBIJGg9dJhln7gezxlnPOM461NpEttGSO4eRWYMLFVu3Ksqcc4L0XskzKbfdRKbTrD43GpWXdzyvM8jFAy9wIuU9o3XEf3sBL7scSKf9FC/9WsIBBGLZNKSJmvJUJnO4DiwGz2XYMqX53jDAhLhVlyZAeaOPYo7OQATYyhSEnb4mF60gsTSlTxqq2J7f4x49i/jc799TSMfOnsWmiK4u32Er93fwVAkzdktpXzi/LlP5dGfzgOHRnj3b/Yx76Spe//K1BOJBN3d3cyaNQu73f4Pf++Dh0f57F2HGYtneO3yWj5y7uy/2d+fMaTkF0OTfLV7hOU+N1+ZVUmNw/as2z5Xxnq6+ONXPos0TVrfdS3XH/oiCSPBuvw6aiNVJGJRTCFAUf/hfhQlj883TtA3gb9oBIdnEoB82k9qahau6VYaJxYRVmzsGRf4VYVVbpW0yDJii5IpEdia/JRUlOA+epT8Tb8i39WFtbaW/vOu4G0jQc6sMGjIdZNOpaAK7pX30jK5lIDwIvQ8nryHjJrmhGU3Ln0Be6sf5srJFt7W8gYOW61s372bdDpNQ2ktCyJVBKN2rHVefOfUYaufMe7x8XE2b97MwYMHEUKwYMECVq1a9dTEE4Zh8NEb7sMYOoxfyVDi9zPniSeoHB6h5qc/wXFy/lIp5Uwrnq3DZDrCIASOeSHcqyqwVnuecbPOZXR23dNDbVuQqjmBF3QeC4Ze4EXJ2NC97PvU76jdtpuxz+aI+ELEh9IEjulUHtaw9uQQUhB3VXBs6SXkUwFKp3dTlTyMZWQAAGtDA56zzmKobTnfHlB48kSYYo+Nd65t5IolVXRNJNnVG6a1wseKxiD7ByJ88Z4j7OmbpqXcy6cunMvKxn/c2uDBw6O8+zd7aanw8as3LXuqBP9cGY6k+eyfDvPwkTHmlHn4yqXzWFRT9M+/COimfEEl8r/H9MgQt33p02QScVa//91YK4JMPbyLXX+6nWBVDRe+76MEqmrQdZ1cLkcul2NycpLDhx9genoLXt8gPt8EimIAFsxsLcnDEB+qIWs/k1wqiMxqgMQiBEJIFsxX8S8oprSlCq/Xi8znid5xJ1M33EB+YABbczPBt78N77nnIlSV2/cM8vHb9nJWhWS+Nsz42NhT+g1MkFZOhI4Qdyap7l/M5obbOHOiijW7vHQ0NJCz2ahKplhZX0fD+vVY6xtJ7Rkj9ugAZjyHbVYRvrNrsVbNVHRPT0+zdetW9u3bh67rtLS00NjYyL2PbMJMR1Fdfi6c3YDji/+D6nJR8/MbsDU2YuYMUnvHSWwdRh9PobgsuJaX4T6tHNX7r918/xEFQy/wokTX4zzw/dfR+JMOEvNsZNYlyNdIpPPkBmmBOFiMdtCD53gEe2ymfbV9wXw8Z63Hs/4sbA0Nz9jnju4pvv3wMXb0hCnz2nnXmU28emkV4WSOrz/QyR37hgi5bXzknFlcvqQa9Tma5SNHxnjHLXuYW+7l5jctx+f856auGyY3bevjWw91YkrJB9bP4k2r67E8bTTCU0F8apLbvvRpYuNjFFVUMtHXw/z157L2dW/BYpt5ysjlJgmHtzAV3kQ4vIVcbgIAIaoYGwsyNhbCZm3ltNPOoKW6mtGPfITUtu34rrwScdW76Do4zfCxCGdcOYuyhpkSsZlOE7n1VqZu/AX62Bj2efMIvf1tuNetwzBNhoaG6Onpobu7m/6BQZAmEkFZaQmpVIp4PM64fZwx3zTLjlzBlGuIR2ffxBnydMomymZK5CUlLEokcG3ZSrazEwCtvBz36afjWnU6UmkguW0MM6XjaAviPbsOS8nMBZdIJNi+fTu7du0im80SMe3Ya+Zxfa2VkQ99GEtVFTU3/AxhD8y0Vtk1hszoWCrduFdW4JxfjLD835/bgqEXeNHy5B3n4PhChrTTSvm3PoKhN9H7o/+Hb6od5/g0llQeqUiysySZhSbGYi/u2oV4vQvweufj9czHav3bR9etXZN8+6Fj7O6bptxnZzo1M/jTW1bX8851TX9TyflceKxjjLffvJdZZW5+/ebl+J3Wv7vtwcEo19/RzqGhGGtnF/PFi9uoDjj/7vb/aVKxKHd89XNMjwxz9tveQ9OyZUSiewiHNxOeepJ44jAAFksRgaJVBAKnEwiuxm4rQ9d1Dh06xNatWxkfH8fj8bD8Fa+gZvt2kjf+AseiRVR+9ztYSkoAMBIJpm/5DeGbbsIIh3EuXUrgbW8j3thAT08PPT099PX1kc/PtO0uLy+noaGB3qybr22eYO3cCv73qoUcPLCPBx56gHw+z0hglAQJGqMNaIZGc3Mza9aseUa3+vzoKIknnyS56UmSW7diJpNgseBcehq2uRdgxINIXeJcVIJ3fS0xm8KO7im2HR/lkb3Hqamu4vuBESY/+xnsbW2UfOKbpNvjZI5OgQBHWwj3qkqsNZ6/W2fyf0HB0Au8aDm241MMfr6Hkt52EnWvwNW3F83IoitWEnWlGCunqH/NB/GUzicWaycWbycWO0AyeQJOjvdit1fPmLt3/ozRe1pRVSdSSp48PslPN3UTclv50Nmz/2VT3dg5zttu3kNTsZtb3rKcItczTT2R1fnWQ53ctLWXoNvG517Zyvnzyv6jAf9cyOejJBNdTE/vIRrfQSSyA8NIIYSGz7uIQPB0goHT8XhaEeLZc+pSSrq6utiyZQs9PT1YrVbmBQJU3PQrvJpG+Zf+h/T+/YR/fQtGLIaxdi2x9WcxZJr09PSQTs8MUhMKhaivr6e+vp66ujqczr+co19t6+Uzdx3m3NYyvn/1IrLpFLfcdQsjx0cAqGmo4ewzz/6n46PIXI7Uvv0kn9xE4olNZI8fJ+kMkV16DZWBuYDgLvL8iiwpi8Lq5hDXpw+Q/d73cK29BmvDOvSJDIpLw7WsHNdp5Wi+/7u0yj+iYOgFXrREwrvY/JX/pfHuneQsbiaDbeQr3YjLd+ApczF/3o9xu2f/zfd0PUE8fphY7MBTRp/JDJ38VMHtav5LKd47H6u1GJBIJMiZYaekNJm5KcinlmfiQSIxZxpzP/UdCZhIJHv7pvnSvYep9Nv5wsUt+J0OrLZiNh3X+ezdnYzG/nml538CXU+STveRSvWQSveSSvWQTvWSSveRz4ef2s7hqCEQmDHwoqLT0DTP8/5fw8PDbNu2jUOHDiGA2rFx6g+2k3S5CC9cyGhREfGTBu71eqmvr6ehoYH6+nq8Xu8/3PeNm3v4wj1HuGBeOd+9ciGaqnCg6wAWxUJL/XNvKRLL5NnVE2Zb1xTbjo1yZDyFRFBhGLwnp7PKHgRM1GAcJdNDav84tuYzQVixVLhwr6zEueA/k1b5RxQMvcCLFtPUeeAP5zJ635vwJvooXr0RfUEXoeCZtLR8C4vlHwf708nmJonH2mcMPnaAaKwdXY/834n/K0wpSOsevO5y/O4KbNZibLZSrLYSbNYSbLYSrLYSrJYQivL8Uz7PhmFkSaf7SKd7SaV6T5r3jInncuPP2NZmK8PhqMXprMfprMPpqMftnoXDUfNv0QIQiUTYvn07e/bseSqF4nA4niqBNzQ0EAgEnvcTy882dfOl+45y8cIKvv3qhc+p7iOZ1dnVG2Zb9xTbu6Y4OBTFlGBVFRbV+FnRGOS0hiALy1wY7fuJb9xJtteC4m9BCAWQM2mV1ZVYa70vmqesgqEXeFGz+/6LiKUOY9Ht5IqzNNS/j7q6d50MqheOlJJMZoBYrJ28PjMxhUABIWZeESeDVJxcnnlv5vOT7wvl5EBUf/vekZE433nkOKaZJ+RMcPYcjdbSHPn8BLnsONncGLncFPDXMSawWkPYrDMGbztp+DPLpU/dCCyWIIqiYZp5MpnBGcM+adzpk6XuTGb4Gfu3WAJPmbXTWYfDWY/TUYfTWYuq/udy+Ol0mhMnThAKhSgtLUVR/vVS7Q8fP8HXH+jk0kWVfOOKBX9j6umcwe6+mRL49u4p2gejT7USWlg9Y+ArGoIsri3Cbvn7TTPT7T2kOybwbJiPpegfN0U9FfwjQ//3FBMKFPgXKJ19CdHew5iqlQVtPyAUXPtv2a8QAoej5t9aAn06Z4TA5prizv3DvHNt47Pm501TJ5efnDH47DjZ3DjZ7NhJw59ZjscP/h3jV7BY/Oh6DCn/0p5e0zw4HfX4fEsoL7vsqRK3w1H3vJ5o/i9xOBzMmzfv37rPd65tQjck3374GKoi+MLFbewbmGZ71xTbuqfYPxAhb0hURTC/ysdbz2hgRUOQpXVFOK3P3eoc8+txzK//t2r/T1EooRc45eh6nJ7eH1BVefX/mfm+2DHNPLncJLncBNns2FPmn8tNPK3UXYfTWY/F8vxTFi8nvv3wMb736HFURWCYEkVAW6WPFQ1BTmsM8oq6wAtqxfRSoVBCL/CiRtM8NDd9/FTLOKUoigW7vRy7vfxUS3nR84H1zZR57XRPJDitIcgr6gPPu7PXy5WCoRcoUOAlhRCCq5f/dz7J/TNObfubAgUKFCjwb6Ng6AUKFCjwMqFg6AUKFCjwMqFg6AUKFCjwMqFg6AUKFCjwMqFg6AUKFCjwMqFg6AUKFCjwMqFg6AUKFCjwMuGUdf0XQkwAfS/w6yFg8t8o5z9JQfupoaD91PBS1f5i1l0rpSx+tg9OmaH/Kwghdv+9sQxe7BS0nxoK2k8NL1XtL1XdhZRLgQIFCrxMKBh6gQIFCrxMeKka+k9PtYB/gYL2U0NB+6nhpar9Jan7JZlDL1CgQIECf8tLtYReoECBAgX+ioKhFyhQoMDLhJecoQshzhVCdAohTgghXjLT3AghqoUQG4UQR4UQh4UQ7zvVmp4PQghVCLFPCHHPqdbyfBBC+IUQtwkhOk4e+xWnWtNzRQjxgZPXyiEhxG+FEC++GYtPIoS4UQgxLoQ49LT3AkKIh4UQx0++Fp1KjX+Pv6P9GyevmXYhxB1CCP8plPiceUkZuhBCBX4AnAe0AFcJIVpOrarnjA58SEo5FzgNeNdLSDvA+4Cjp1rEC+C7wANSyjnAAl4iv0EIUQm8F1gqpWwDVODKU6vqH/JL4Ny/eu/jwKNSymbg0ZPrL0Z+yd9qfxhok1LOB44B1/+nRb0QXlKGDiwDTkgpu6WUOeB3wMWnWNNzQko5IqXce3I5zoyxVJ5aVc8NIUQVcAFww6nW8nwQQniBM4CfA0gpc1LKyCkV9fzQAIcQQgOcwPAp1vN3kVJuAsJ/9fbFwE0nl28CLvlPanquPJt2KeVDUkr95Op2oOo/LuwF8FIz9Epg4Gnrg7xETPHpCCHqgEXAjlMs5bnyHeCjgHmKdTxfGoAJ4Bcn00U3CCFcp1rUc0FKOQR8E+gHRoColPKhU6vqeVMqpRyBmQINUHKK9bxQ3gTcf6pFPBdeaoYunuW9l1S7SyGEG7gdeL+UMnaq9fwzhBAXAuNSyj2nWssLQAMWAz+SUi4Ckrx4H/ufwcl888VAPVABuIQQrz21qv77EEJ8kpl06S2nWstz4aVm6INA9dPWq3gRP4b+NUIICzNmfouU8o+nWs9zZBVwkRCil5kU15lCiF+fWknPmUFgUEr55yeh25gx+JcC64EeKeWElDIP/BFYeYo1PV/GhBDlACdfx0+xnueFEOL1wIXANfIl0mHnpWbou4BmIUS9EMLKTCXRn06xpueEEEIwk8s9KqX89qnW81yRUl4vpaySUtYxc7wfk1K+JEqKUspRYEAIMfvkW2cBR06hpOdDP3CaEMJ58to5i5dIhe7T+BPw+pPLrwfuOoVanhdCiHOBjwEXSSlTp1rPc+UlZegnKyneDTzIzMV9q5Ty8KlV9ZxZBVzLTAl3/8m/80+1qP8C3gPcIoRoBxYCXz61cp4bJ58qbgP2AgeZidUXbXd0IcRvgW3AbCHEoBDizcBXgQ1CiOPAhpPrLzr+jvb/BTzAwydj9cenVORzpND1v0CBAgVeJrykSugFChQoUODvUzD0AgUKFHiZUDD0AgUKFHiZUDD0AgUKFHiZUDD0AgUKFHiZUDD0AgUKFHiZUDD0AgUKFHiZ8P8BfGvkEgqqaBYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot test samples\n",
    "plot_test = np.vstack([data[1].reshape((1, -1)) for data in test_data])\n",
    "out = plt.plot(plot_test.transpose())\n",
    "plt.show()\n",
    "# save full data\n",
    "np.save(f\"{cache_path}/test_data.npy\", plot_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create PyTorch data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data(Dataset):\n",
    "  def __init__(self, data, flag='train', size=None):\n",
    "    # size [seq_len, label_len, pred_len]\n",
    "    # info\n",
    "    self.seq_len = size[0]\n",
    "    self.label_len = size[1]\n",
    "    self.pred_len = size[2]\n",
    "    \n",
    "    # init\n",
    "    self.data = data\n",
    "    len_segs = np.array([len(subj_seg[1]) for subj_seg in self.data])\n",
    "    len_segs = len_segs - self.seq_len - self.pred_len + 1\n",
    "    self.len_segs = np.insert(np.cumsum(len_segs), 0, 0)\n",
    "  \n",
    "  def __getitem__(self, index):\n",
    "\n",
    "    idx_seg = np.argmax(self.len_segs > index) - 1\n",
    "    seg = self.data[idx_seg]\n",
    "\n",
    "    s_begin = index - self.len_segs[idx_seg]\n",
    "    s_end = s_begin + self.seq_len\n",
    "    r_begin = s_end - self.label_len\n",
    "    r_end = r_begin + self.label_len + self.pred_len\n",
    "\n",
    "    subj_id = seg[0]\n",
    "    seq_x = seg[1][s_begin:s_end]\n",
    "    seq_y = seg[1][r_begin:r_end]\n",
    "    seq_x_mark = seg[2][s_begin:s_end]\n",
    "    seq_y_mark = seg[2][r_begin:r_end]\n",
    "\n",
    "    return subj_id, seq_x, seq_y, seq_x_mark, seq_y_mark\n",
    "  \n",
    "  def __len__(self):\n",
    "    return self.len_segs[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_seq, len_label, len_pred = 4, 2, 2\n",
    "\n",
    "train_data = Data(train_data, size=[len_seq, len_label, len_pred])\n",
    "val_data = Data(val_data, size=[len_seq, len_label, len_pred])\n",
    "test_data = Data(test_data, size=[len_seq, len_label, len_pred])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0')\n",
    "model = Gluformer(d_model=128, \n",
    "                    n_heads=4, \n",
    "                    d_fcn=256, \n",
    "                    r_drop=0.3, \n",
    "                    activ=\"relu\", \n",
    "                    num_enc_layers=2, \n",
    "                    num_dec_layers=1,\n",
    "                    distil=True,\n",
    "                    len_seq=len_seq, \n",
    "                    len_pred=len_pred,\n",
    "                    num_features=1)\n",
    "model.train()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with the infinite mixture objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = os.path.join(save_path, \"model_inf_mixture.pth\")\n",
    "num_samples = 3 # number of samples for MC estimate\n",
    "batch_size = 256 # batch size for optimization\n",
    "collate_fn_custom = modify_collate(num_samples)\n",
    "\n",
    "train_data_loader = DataLoader(train_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=True, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True, \n",
    "                                collate_fn = collate_fn_custom)\n",
    "\n",
    "val_data_loader = DataLoader(val_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=True, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True, \n",
    "                                collate_fn = collate_fn_custom)\n",
    "\n",
    "test_data_loader = DataLoader(test_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=False, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True,\n",
    "                                collate_fn = collate_fn_custom)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t iters: 10 / 70, epoch: 1 | loss: -2.0519629\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08298508 0.04760794 0.12853129]\n",
      "\t speed: 0.0445s/iter; left time: 311.1980s\n",
      "\t iters: 20 / 70, epoch: 1 | loss: -2.3055284\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03383549 0.01589167 0.11618093]\n",
      "\t speed: 0.0292s/iter; left time: 204.0158s\n",
      "\t iters: 30 / 70, epoch: 1 | loss: -2.4904387\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.17945786 0.15005569 0.15005569]\n",
      "\t speed: 0.0284s/iter; left time: 198.2775s\n",
      "\t iters: 40 / 70, epoch: 1 | loss: -2.6184139\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0402912  0.05934384 0.04228688]\n",
      "\t speed: 0.0262s/iter; left time: 182.3129s\n",
      "\t iters: 50 / 70, epoch: 1 | loss: -2.7226963\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.014023   0.14086013 0.04904477]\n",
      "\t speed: 0.0288s/iter; left time: 200.5011s\n",
      "\t iters: 60 / 70, epoch: 1 | loss: -2.4534197\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02123966 0.06094038 0.14437614]\n",
      "\t speed: 0.0286s/iter; left time: 198.5523s\n",
      "\t iters: 70 / 70, epoch: 1 | loss: -3.1048667\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05501182 0.02591334 0.01064554]\n",
      "\t speed: 0.0283s/iter; left time: 196.4209s\n",
      "Validation loss descreased: inf -> -3.0584401289621987\n",
      "Epoch: 1 Time: 2.2106637954711914 Steps: 70\n",
      "Train Loss: -2.2242823 | Val Loss: -3.0584401\n",
      "\t iters: 10 / 70, epoch: 2 | loss: -3.1951180\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.09883834 0.03076491 0.0414615 ]\n",
      "\t speed: 0.0281s/iter; left time: 194.6261s\n",
      "\t iters: 20 / 70, epoch: 2 | loss: -3.1066046\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02304384 0.03512987 0.03744164]\n",
      "\t speed: 0.0294s/iter; left time: 203.4931s\n",
      "\t iters: 30 / 70, epoch: 2 | loss: -3.1522303\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01574028 0.02071819 0.03170416]\n",
      "\t speed: 0.0287s/iter; left time: 198.1192s\n",
      "\t iters: 40 / 70, epoch: 2 | loss: -2.8818016\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06018776 0.0224065  0.02064945]\n",
      "\t speed: 0.0288s/iter; left time: 198.5872s\n",
      "\t iters: 50 / 70, epoch: 2 | loss: -3.3774691\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01059678 0.03733165 0.13205868]\n",
      "\t speed: 0.0401s/iter; left time: 276.2654s\n",
      "\t iters: 60 / 70, epoch: 2 | loss: -3.2058523\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0233108  0.02207465 0.13062736]\n",
      "\t speed: 0.0280s/iter; left time: 192.7123s\n",
      "\t iters: 70 / 70, epoch: 2 | loss: -3.3473492\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02035039 0.0298925  0.01514485]\n",
      "\t speed: 0.0284s/iter; left time: 194.7264s\n",
      "Validation loss descreased: -3.0584401289621987 -> -3.0779407024383545\n",
      "Epoch: 2 Time: 2.1816744804382324 Steps: 70\n",
      "Train Loss: -3.1549055 | Val Loss: -3.0779407\n",
      "\t iters: 10 / 70, epoch: 3 | loss: -3.3947606\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01412226 0.12687995 0.0286699 ]\n",
      "\t speed: 0.0282s/iter; left time: 193.5228s\n",
      "\t iters: 20 / 70, epoch: 3 | loss: -3.6363420\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01210378 0.12465751 0.12465751]\n",
      "\t speed: 0.0272s/iter; left time: 186.2002s\n",
      "\t iters: 30 / 70, epoch: 3 | loss: -3.3754487\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04925017 0.02879731 0.02259634]\n",
      "\t speed: 0.0274s/iter; left time: 187.4338s\n",
      "\t iters: 40 / 70, epoch: 3 | loss: -3.7202063\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02356955 0.01899086 0.12077033]\n",
      "\t speed: 0.0280s/iter; left time: 191.1062s\n",
      "\t iters: 50 / 70, epoch: 3 | loss: -3.7734418\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01572915 0.02049729 0.11875903]\n",
      "\t speed: 0.0307s/iter; left time: 209.1332s\n",
      "\t iters: 60 / 70, epoch: 3 | loss: -3.7740521\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02908799 0.11730692 0.02213459]\n",
      "\t speed: 0.0279s/iter; left time: 189.6213s\n",
      "\t iters: 70 / 70, epoch: 3 | loss: -3.7774017\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01223159 0.01186627 0.1153034 ]\n",
      "\t speed: 0.0303s/iter; left time: 205.8573s\n",
      "Validation loss descreased: -3.0779407024383545 -> -3.6933323542277017\n",
      "Epoch: 3 Time: 2.0677666664123535 Steps: 70\n",
      "Train Loss: -3.5093085 | Val Loss: -3.6933324\n",
      "\t iters: 10 / 70, epoch: 4 | loss: -3.6680815\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.11387171 0.01294285 0.01294786]\n",
      "\t speed: 0.0263s/iter; left time: 178.2942s\n",
      "\t iters: 20 / 70, epoch: 4 | loss: -3.4051287\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01254092 0.11301829 0.01383695]\n",
      "\t speed: 0.0276s/iter; left time: 186.7258s\n",
      "\t iters: 30 / 70, epoch: 4 | loss: -3.8042386\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.11098306 0.02137117 0.01249676]\n",
      "\t speed: 0.0363s/iter; left time: 245.7615s\n",
      "\t iters: 40 / 70, epoch: 4 | loss: -4.0073681\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.11002533 0.01874756 0.11002533]\n",
      "\t speed: 0.0280s/iter; left time: 189.1025s\n",
      "\t iters: 50 / 70, epoch: 4 | loss: -3.7930193\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.10909954 0.01450035 0.10909954]\n",
      "\t speed: 0.0273s/iter; left time: 184.1399s\n",
      "\t iters: 60 / 70, epoch: 4 | loss: -3.9296408\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.10693654 0.02783906 0.01223446]\n",
      "\t speed: 0.0287s/iter; left time: 193.4635s\n",
      "\t iters: 70 / 70, epoch: 4 | loss: -3.8414097\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.10615663 0.00927181 0.01006607]\n",
      "\t speed: 0.0301s/iter; left time: 201.9860s\n",
      "Validation loss descreased: -3.6933323542277017 -> -3.913264274597168\n",
      "Epoch: 4 Time: 2.120131492614746 Steps: 70\n",
      "Train Loss: -3.7424381 | Val Loss: -3.9132643\n",
      "\t iters: 10 / 70, epoch: 5 | loss: -3.8175926\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.10535204 0.00567806 0.10535204]\n",
      "\t speed: 0.0297s/iter; left time: 199.5568s\n",
      "\t iters: 20 / 70, epoch: 5 | loss: -3.4125872\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.1041998  0.00986272 0.1041998 ]\n",
      "\t speed: 0.0286s/iter; left time: 191.6077s\n",
      "\t iters: 30 / 70, epoch: 5 | loss: -4.2283988\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.10322406 0.02315788 0.01152581]\n",
      "\t speed: 0.0289s/iter; left time: 193.1948s\n",
      "\t iters: 40 / 70, epoch: 5 | loss: -3.6680036\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01350392 0.10204961 0.00531417]\n",
      "\t speed: 0.0278s/iter; left time: 185.7061s\n",
      "\t iters: 50 / 70, epoch: 5 | loss: -3.9000645\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00946629 0.10070898 0.10070898]\n",
      "\t speed: 0.0290s/iter; left time: 193.3103s\n",
      "\t iters: 60 / 70, epoch: 5 | loss: -4.1107826\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.009341   0.00870456 0.00868992]\n",
      "\t speed: 0.0286s/iter; left time: 190.6942s\n",
      "\t iters: 70 / 70, epoch: 5 | loss: -3.9672470\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01036596 0.09863597 0.00957745]\n",
      "\t speed: 0.0308s/iter; left time: 205.0329s\n",
      "Validation loss descreased: -3.913264274597168 -> -4.0491943359375\n",
      "Epoch: 5 Time: 2.107569694519043 Steps: 70\n",
      "Train Loss: -3.8868163 | Val Loss: -4.0491943\n",
      "\t iters: 10 / 70, epoch: 6 | loss: -4.0702467\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01813728 0.01009536 0.0202354 ]\n",
      "\t speed: 0.0262s/iter; left time: 173.9588s\n",
      "\t iters: 20 / 70, epoch: 6 | loss: -3.4658842\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.09674197 0.00909207 0.01940132]\n",
      "\t speed: 0.0367s/iter; left time: 243.2420s\n",
      "\t iters: 30 / 70, epoch: 6 | loss: -3.8088527\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.09587231 0.09587231 0.09587231]\n",
      "\t speed: 0.0282s/iter; left time: 186.7613s\n",
      "\t iters: 40 / 70, epoch: 6 | loss: -4.2489038\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.09488922 0.01795315 0.00618952]\n",
      "\t speed: 0.0278s/iter; left time: 183.8224s\n",
      "\t iters: 50 / 70, epoch: 6 | loss: -4.2700615\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.09450305 0.00834449 0.09450305]\n",
      "\t speed: 0.0279s/iter; left time: 184.0681s\n",
      "\t iters: 60 / 70, epoch: 6 | loss: -3.6855450\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01290837 0.09300801 0.00645225]\n",
      "\t speed: 0.0301s/iter; left time: 198.4371s\n",
      "\t iters: 70 / 70, epoch: 6 | loss: -4.3112116\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01526089 0.01468176 0.0201438 ]\n",
      "\t speed: 0.0315s/iter; left time: 206.9761s\n",
      "Validation loss descreased: -4.0491943359375 -> -4.170547008514404\n",
      "Epoch: 6 Time: 2.1542534828186035 Steps: 70\n",
      "Train Loss: -3.9832120 | Val Loss: -4.1705470\n",
      "\t iters: 10 / 70, epoch: 7 | loss: -4.1350884\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00358548 0.00752512 0.00733828]\n",
      "\t speed: 0.0298s/iter; left time: 195.8130s\n",
      "\t iters: 20 / 70, epoch: 7 | loss: -3.9728427\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01386254 0.09105687 0.09105687]\n",
      "\t speed: 0.0325s/iter; left time: 213.2540s\n",
      "\t iters: 30 / 70, epoch: 7 | loss: -3.8508830\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00580563 0.09027137 0.01287814]\n",
      "\t speed: 0.0319s/iter; left time: 209.0085s\n",
      "\t iters: 40 / 70, epoch: 7 | loss: -4.0241928\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08925974 0.08925974 0.00973075]\n",
      "\t speed: 0.0284s/iter; left time: 185.7221s\n",
      "\t iters: 50 / 70, epoch: 7 | loss: -4.0672493\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08793205 0.02210755 0.0166455 ]\n",
      "\t speed: 0.0295s/iter; left time: 192.5635s\n",
      "\t iters: 60 / 70, epoch: 7 | loss: -4.2231855\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00697091 0.01425155 0.02134228]\n",
      "\t speed: 0.0284s/iter; left time: 185.2021s\n",
      "\t iters: 70 / 70, epoch: 7 | loss: -4.3061190\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01264568 0.01632188 0.01219552]\n",
      "\t speed: 0.0392s/iter; left time: 255.3510s\n",
      "Validation loss descreased: -4.170547008514404 -> -4.173453013102214\n",
      "Epoch: 7 Time: 2.2637031078338623 Steps: 70\n",
      "Train Loss: -4.0196916 | Val Loss: -4.1734530\n",
      "\t iters: 10 / 70, epoch: 8 | loss: -3.9542301\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01454298 0.0270247  0.00890574]\n",
      "\t speed: 0.0291s/iter; left time: 188.9551s\n",
      "\t iters: 20 / 70, epoch: 8 | loss: -4.3329258\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0059007  0.08580099 0.01150777]\n",
      "\t speed: 0.0302s/iter; left time: 195.7741s\n",
      "\t iters: 30 / 70, epoch: 8 | loss: -4.0551367\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00607521 0.01018919 0.08446954]\n",
      "\t speed: 0.0306s/iter; left time: 198.4935s\n",
      "\t iters: 40 / 70, epoch: 8 | loss: -3.9211464\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00595361 0.0067604  0.01526212]\n",
      "\t speed: 0.0306s/iter; left time: 198.0729s\n",
      "\t iters: 50 / 70, epoch: 8 | loss: -3.9914858\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00533044 0.00401554 0.08395579]\n",
      "\t speed: 0.0298s/iter; left time: 192.7802s\n",
      "\t iters: 60 / 70, epoch: 8 | loss: -4.4161620\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08269431 0.08269431 0.00468432]\n",
      "\t speed: 0.0294s/iter; left time: 189.3525s\n",
      "\t iters: 70 / 70, epoch: 8 | loss: -4.1432047\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01304687 0.00669938 0.01827781]\n",
      "\t speed: 0.0309s/iter; left time: 198.9828s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 8 Time: 2.1645193099975586 Steps: 70\n",
      "Train Loss: -4.1168798 | Val Loss: -4.1085676\n",
      "\t iters: 10 / 70, epoch: 9 | loss: -3.2754893\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00943759 0.00926013 0.00464984]\n",
      "\t speed: 0.0304s/iter; left time: 195.2428s\n",
      "\t iters: 20 / 70, epoch: 9 | loss: -4.1566434\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01109116 0.08113778 0.08113778]\n",
      "\t speed: 0.0310s/iter; left time: 198.9646s\n",
      "\t iters: 30 / 70, epoch: 9 | loss: -4.0162382\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08061003 0.00848865 0.00421735]\n",
      "\t speed: 0.0310s/iter; left time: 198.9858s\n",
      "\t iters: 40 / 70, epoch: 9 | loss: -4.2734089\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01803644 0.07958231 0.07958231]\n",
      "\t speed: 0.0309s/iter; left time: 197.8550s\n",
      "\t iters: 50 / 70, epoch: 9 | loss: -4.1991148\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01757891 0.01095744 0.00759156]\n",
      "\t speed: 0.0424s/iter; left time: 270.7780s\n",
      "\t iters: 60 / 70, epoch: 9 | loss: -4.0148468\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07786946 0.00479836 0.00413949]\n",
      "\t speed: 0.0318s/iter; left time: 203.1413s\n",
      "\t iters: 70 / 70, epoch: 9 | loss: -4.4501328\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07771333 0.01700357 0.01942915]\n",
      "\t speed: 0.0308s/iter; left time: 196.4287s\n",
      "Validation loss descreased: -4.173453013102214 -> -4.290862878163655\n",
      "Epoch: 9 Time: 2.3523402214050293 Steps: 70\n",
      "Train Loss: -4.1652198 | Val Loss: -4.2908629\n",
      "\t iters: 10 / 70, epoch: 10 | loss: -4.1359258\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00660818 0.00932589 0.0770865 ]\n",
      "\t speed: 0.0291s/iter; left time: 185.2444s\n",
      "\t iters: 20 / 70, epoch: 10 | loss: -4.1219354\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00397264 0.07654935 0.00784731]\n",
      "\t speed: 0.0295s/iter; left time: 187.0537s\n",
      "\t iters: 30 / 70, epoch: 10 | loss: -4.2144327\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00783696 0.01825388 0.00570264]\n",
      "\t speed: 0.0320s/iter; left time: 202.8451s\n",
      "\t iters: 40 / 70, epoch: 10 | loss: -4.2538705\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00434408 0.00759315 0.00645251]\n",
      "\t speed: 0.0305s/iter; left time: 193.1117s\n",
      "\t iters: 50 / 70, epoch: 10 | loss: -3.8832493\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07432364 0.07432364 0.00794829]\n",
      "\t speed: 0.0313s/iter; left time: 198.1546s\n",
      "\t iters: 60 / 70, epoch: 10 | loss: -4.3336267\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07391401 0.01183827 0.00913802]\n",
      "\t speed: 0.0289s/iter; left time: 182.1678s\n",
      "\t iters: 70 / 70, epoch: 10 | loss: -4.4809990\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0736382  0.00655403 0.00832697]\n",
      "\t speed: 0.0305s/iter; left time: 192.2950s\n",
      "Validation loss descreased: -4.290862878163655 -> -4.38112751642863\n",
      "Epoch: 10 Time: 2.1955220699310303 Steps: 70\n",
      "Train Loss: -4.2104482 | Val Loss: -4.3811275\n",
      "\t iters: 10 / 70, epoch: 11 | loss: -4.2482190\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07326604 0.07326604 0.07326604]\n",
      "\t speed: 0.0293s/iter; left time: 184.5555s\n",
      "\t iters: 20 / 70, epoch: 11 | loss: -4.2755995\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00546224 0.01035113 0.07265341]\n",
      "\t speed: 0.0281s/iter; left time: 176.5871s\n",
      "\t iters: 30 / 70, epoch: 11 | loss: -4.2812090\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00636193 0.00853998 0.01541515]\n",
      "\t speed: 0.0286s/iter; left time: 179.6610s\n",
      "\t iters: 40 / 70, epoch: 11 | loss: -4.3328304\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00725876 0.07169503 0.07169503]\n",
      "\t speed: 0.0392s/iter; left time: 245.2620s\n",
      "\t iters: 50 / 70, epoch: 11 | loss: -4.2797194\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07102362 0.00411185 0.07102362]\n",
      "\t speed: 0.0280s/iter; left time: 175.2383s\n",
      "\t iters: 60 / 70, epoch: 11 | loss: -4.3169727\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.07073683 0.01818337 0.00604476]\n",
      "\t speed: 0.0284s/iter; left time: 176.9735s\n",
      "\t iters: 70 / 70, epoch: 11 | loss: -4.1176987\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01125395 0.00997935 0.01898956]\n",
      "\t speed: 0.0297s/iter; left time: 185.0706s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 11 Time: 2.1683449745178223 Steps: 70\n",
      "Train Loss: -4.1990323 | Val Loss: -4.0426586\n",
      "\t iters: 10 / 70, epoch: 12 | loss: -4.4093428\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0693344  0.01073102 0.0693344 ]\n",
      "\t speed: 0.0292s/iter; left time: 181.6607s\n",
      "\t iters: 20 / 70, epoch: 12 | loss: -4.2270708\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06893056 0.06893056 0.01146855]\n",
      "\t speed: 0.0303s/iter; left time: 188.0154s\n",
      "\t iters: 30 / 70, epoch: 12 | loss: -4.0546627\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00720681 0.01104886 0.00317765]\n",
      "\t speed: 0.0307s/iter; left time: 190.4388s\n",
      "\t iters: 40 / 70, epoch: 12 | loss: -4.5305314\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0072818  0.06856389 0.01206365]\n",
      "\t speed: 0.0281s/iter; left time: 173.9437s\n",
      "\t iters: 50 / 70, epoch: 12 | loss: -4.5049281\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00978427 0.00610293 0.00566513]\n",
      "\t speed: 0.0286s/iter; left time: 176.4891s\n",
      "\t iters: 60 / 70, epoch: 12 | loss: -4.0866513\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01160678 0.01541402 0.01429036]\n",
      "\t speed: 0.0277s/iter; left time: 170.8506s\n",
      "\t iters: 70 / 70, epoch: 12 | loss: -4.3654633\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06717881 0.00861158 0.0095398 ]\n",
      "\t speed: 0.0281s/iter; left time: 173.0557s\n",
      "Validation loss descreased: -4.38112751642863 -> -4.417974948883057\n",
      "Epoch: 12 Time: 2.091651439666748 Steps: 70\n",
      "Train Loss: -4.2756556 | Val Loss: -4.4179749\n",
      "\t iters: 10 / 70, epoch: 13 | loss: -4.4340153\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06673251 0.00441242 0.06673251]\n",
      "\t speed: 0.0265s/iter; left time: 163.1742s\n",
      "\t iters: 20 / 70, epoch: 13 | loss: -4.1618743\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00591503 0.00708363 0.01657583]\n",
      "\t speed: 0.0384s/iter; left time: 235.5158s\n",
      "\t iters: 30 / 70, epoch: 13 | loss: -4.1694660\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00862129 0.00790329 0.00793556]\n",
      "\t speed: 0.0282s/iter; left time: 172.6649s\n",
      "\t iters: 40 / 70, epoch: 13 | loss: -4.2785802\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0076608  0.00573346 0.00712783]\n",
      "\t speed: 0.0284s/iter; left time: 173.9268s\n",
      "\t iters: 50 / 70, epoch: 13 | loss: -4.5106597\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06486387 0.00548918 0.00779012]\n",
      "\t speed: 0.0279s/iter; left time: 170.7471s\n",
      "\t iters: 60 / 70, epoch: 13 | loss: -4.3165722\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01114854 0.00809054 0.00448794]\n",
      "\t speed: 0.0287s/iter; left time: 174.8107s\n",
      "\t iters: 70 / 70, epoch: 13 | loss: -4.4231730\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00888707 0.00710141 0.01475007]\n",
      "\t speed: 0.0287s/iter; left time: 174.9823s\n",
      "Validation loss descreased: -4.417974948883057 -> -4.51267655690511\n",
      "Epoch: 13 Time: 2.13489031791687 Steps: 70\n",
      "Train Loss: -4.3060842 | Val Loss: -4.5126766\n",
      "\t iters: 10 / 70, epoch: 14 | loss: -4.3510919\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05896871 0.06366271 0.02583725]\n",
      "\t speed: 0.0279s/iter; left time: 169.7801s\n",
      "\t iters: 20 / 70, epoch: 14 | loss: -4.1351633\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00854699 0.01175308 0.04477235]\n",
      "\t speed: 0.0291s/iter; left time: 176.7938s\n",
      "\t iters: 30 / 70, epoch: 14 | loss: -4.1597505\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00794119 0.0079632  0.00346192]\n",
      "\t speed: 0.0285s/iter; left time: 172.5664s\n",
      "\t iters: 40 / 70, epoch: 14 | loss: -4.2552271\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.06267253 0.06267253 0.00408815]\n",
      "\t speed: 0.0285s/iter; left time: 172.5541s\n",
      "\t iters: 50 / 70, epoch: 14 | loss: -4.3406639\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00227289 0.06261244 0.00156516]\n",
      "\t speed: 0.0304s/iter; left time: 183.4857s\n",
      "\t iters: 60 / 70, epoch: 14 | loss: -4.5010271\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.08716714 0.00887735 0.00416217]\n",
      "\t speed: 0.0324s/iter; left time: 195.4007s\n",
      "\t iters: 70 / 70, epoch: 14 | loss: -4.1945210\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01461555 0.00745065 0.00422711]\n",
      "\t speed: 0.0316s/iter; left time: 190.2904s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 14 Time: 2.244072198867798 Steps: 70\n",
      "Train Loss: -4.3579756 | Val Loss: -4.4131112\n",
      "\t iters: 10 / 70, epoch: 15 | loss: -4.6070771\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00216403 0.00837571 0.02020117]\n",
      "\t speed: 0.0288s/iter; left time: 173.3466s\n",
      "\t iters: 20 / 70, epoch: 15 | loss: -4.6666374\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02806823 0.00265406 0.00223347]\n",
      "\t speed: 0.0298s/iter; left time: 178.8346s\n",
      "\t iters: 30 / 70, epoch: 15 | loss: -4.1776800\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01155977 0.02435056 0.01113794]\n",
      "\t speed: 0.0313s/iter; left time: 187.4643s\n",
      "\t iters: 40 / 70, epoch: 15 | loss: -4.3372192\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01351841 0.0397592  0.00382792]\n",
      "\t speed: 0.0320s/iter; left time: 191.4422s\n",
      "\t iters: 50 / 70, epoch: 15 | loss: -4.1146178\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0102234  0.00700235 0.00503881]\n",
      "\t speed: 0.0314s/iter; left time: 187.4919s\n",
      "\t iters: 60 / 70, epoch: 15 | loss: -4.5242882\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00518595 0.00199413 0.00396586]\n",
      "\t speed: 0.0320s/iter; left time: 190.9899s\n",
      "\t iters: 70 / 70, epoch: 15 | loss: -4.5053520\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01698564 0.0003375  0.11998832]\n",
      "\t speed: 0.0306s/iter; left time: 181.8844s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 15 Time: 2.216489553451538 Steps: 70\n",
      "Train Loss: -4.4161242 | Val Loss: -4.4254440\n",
      "\t iters: 10 / 70, epoch: 16 | loss: -4.3214817\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00254191 0.00517279 0.00061665]\n",
      "\t speed: 0.0280s/iter; left time: 166.4250s\n",
      "\t iters: 20 / 70, epoch: 16 | loss: -4.1794100\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01833939 0.00476424 0.02475059]\n",
      "\t speed: 0.0292s/iter; left time: 173.1732s\n",
      "\t iters: 30 / 70, epoch: 16 | loss: -4.3846750\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00492899 0.01656914 0.01628897]\n",
      "\t speed: 0.0292s/iter; left time: 173.1785s\n",
      "\t iters: 40 / 70, epoch: 16 | loss: -4.4323659\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00693972 0.00877092 0.00998147]\n",
      "\t speed: 0.0298s/iter; left time: 176.0212s\n",
      "\t iters: 50 / 70, epoch: 16 | loss: -4.3292742\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00226606 0.00318875 0.04494346]\n",
      "\t speed: 0.0314s/iter; left time: 185.3168s\n",
      "\t iters: 60 / 70, epoch: 16 | loss: -4.6777043\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00484423 0.00054442 0.00094163]\n",
      "\t speed: 0.0396s/iter; left time: 233.4090s\n",
      "\t iters: 70 / 70, epoch: 16 | loss: -4.6782146\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02161673 0.03817885 0.01174093]\n",
      "\t speed: 0.0307s/iter; left time: 180.8058s\n",
      "Validation loss descreased: -4.51267655690511 -> -4.584070205688477\n",
      "Epoch: 16 Time: 2.2541258335113525 Steps: 70\n",
      "Train Loss: -4.5493826 | Val Loss: -4.5840702\n",
      "\t iters: 10 / 70, epoch: 17 | loss: -4.7304068\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02834887 0.00182472 0.00862155]\n",
      "\t speed: 0.0300s/iter; left time: 176.2872s\n",
      "\t iters: 20 / 70, epoch: 17 | loss: -4.6207371\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01558889 0.02258014 0.02011005]\n",
      "\t speed: 0.0312s/iter; left time: 183.0283s\n",
      "\t iters: 30 / 70, epoch: 17 | loss: -4.7636561\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04003314 0.00608004 0.02299558]\n",
      "\t speed: 0.0319s/iter; left time: 186.6991s\n",
      "\t iters: 40 / 70, epoch: 17 | loss: -4.6997061\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03600161 0.00136755 0.00260828]\n",
      "\t speed: 0.0284s/iter; left time: 165.7512s\n",
      "\t iters: 50 / 70, epoch: 17 | loss: -4.3298988\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02632258 0.01983249 0.00516825]\n",
      "\t speed: 0.0284s/iter; left time: 165.5959s\n",
      "\t iters: 60 / 70, epoch: 17 | loss: -4.8178310\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00085744 0.00442629 0.00245781]\n",
      "\t speed: 0.0284s/iter; left time: 165.3187s\n",
      "\t iters: 70 / 70, epoch: 17 | loss: -4.7936831\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01587974 0.00428873 0.02304985]\n",
      "\t speed: 0.0284s/iter; left time: 165.0220s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 17 Time: 2.1238210201263428 Steps: 70\n",
      "Train Loss: -4.6280454 | Val Loss: -4.4290214\n",
      "\t iters: 10 / 70, epoch: 18 | loss: -4.8427296\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00085481 0.01558552 0.0014089 ]\n",
      "\t speed: 0.0280s/iter; left time: 162.2081s\n",
      "\t iters: 20 / 70, epoch: 18 | loss: -4.7852383\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05586852 0.00157056 0.00352169]\n",
      "\t speed: 0.0294s/iter; left time: 170.0547s\n",
      "\t iters: 30 / 70, epoch: 18 | loss: -4.7588911\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00037218 0.00132739 0.00768606]\n",
      "\t speed: 0.0303s/iter; left time: 175.3870s\n",
      "\t iters: 40 / 70, epoch: 18 | loss: -4.9468918\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00192066 0.05831674 0.0031096 ]\n",
      "\t speed: 0.0418s/iter; left time: 241.5008s\n",
      "\t iters: 50 / 70, epoch: 18 | loss: -4.7886167\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00312921 0.05786617 0.00509081]\n",
      "\t speed: 0.0286s/iter; left time: 164.6928s\n",
      "\t iters: 60 / 70, epoch: 18 | loss: -4.4305043\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05826656 0.01638403 0.00976841]\n",
      "\t speed: 0.0311s/iter; left time: 178.9188s\n",
      "\t iters: 70 / 70, epoch: 18 | loss: -4.5463543\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04575359 0.08974221 0.02975705]\n",
      "\t speed: 0.0309s/iter; left time: 177.5042s\n",
      "Validation loss descreased: -4.584070205688477 -> -4.7363050778706866\n",
      "Epoch: 18 Time: 2.271347999572754 Steps: 70\n",
      "Train Loss: -4.6710466 | Val Loss: -4.7363051\n",
      "\t iters: 10 / 70, epoch: 19 | loss: -4.9021330\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0398043  0.01509528 0.00629403]\n",
      "\t speed: 0.0289s/iter; left time: 165.4609s\n",
      "\t iters: 20 / 70, epoch: 19 | loss: -5.0483990\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00548326 0.04626865 0.03796785]\n",
      "\t speed: 0.0296s/iter; left time: 169.2116s\n",
      "\t iters: 30 / 70, epoch: 19 | loss: -4.8977284\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0008973  0.00277793 0.00181032]\n",
      "\t speed: 0.0311s/iter; left time: 177.8637s\n",
      "\t iters: 40 / 70, epoch: 19 | loss: -4.7009263\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00172474 0.00103978 0.00876511]\n",
      "\t speed: 0.0304s/iter; left time: 173.0825s\n",
      "\t iters: 50 / 70, epoch: 19 | loss: -4.8175249\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00200553 0.00152636 0.00339121]\n",
      "\t speed: 0.0306s/iter; left time: 174.2134s\n",
      "\t iters: 60 / 70, epoch: 19 | loss: -4.4049697\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00033208 0.038289   0.0094633 ]\n",
      "\t speed: 0.0293s/iter; left time: 166.4723s\n",
      "\t iters: 70 / 70, epoch: 19 | loss: -4.8371458\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01663165 0.03945578 0.02416428]\n",
      "\t speed: 0.0316s/iter; left time: 179.3836s\n",
      "Validation loss descreased: -4.7363050778706866 -> -4.843667507171631\n",
      "Epoch: 19 Time: 2.1903891563415527 Steps: 70\n",
      "Train Loss: -4.7398249 | Val Loss: -4.8436675\n",
      "\t iters: 10 / 70, epoch: 20 | loss: -4.6163034\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00592708 0.00545494 0.0100636 ]\n",
      "\t speed: 0.0292s/iter; left time: 165.1077s\n",
      "\t iters: 20 / 70, epoch: 20 | loss: -4.6858644\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00350357 0.02017632 0.00832224]\n",
      "\t speed: 0.0285s/iter; left time: 161.1068s\n",
      "\t iters: 30 / 70, epoch: 20 | loss: -4.8029509\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0157708  0.0247744  0.03873455]\n",
      "\t speed: 0.0411s/iter; left time: 231.9230s\n",
      "\t iters: 40 / 70, epoch: 20 | loss: -5.0627651\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0085761  0.0021357  0.00303533]\n",
      "\t speed: 0.0307s/iter; left time: 172.8308s\n",
      "\t iters: 50 / 70, epoch: 20 | loss: -4.6372552\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04439884 0.05531522 0.04547765]\n",
      "\t speed: 0.0317s/iter; left time: 178.2906s\n",
      "\t iters: 60 / 70, epoch: 20 | loss: -4.7046261\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02437174 0.05532155 0.03877747]\n",
      "\t speed: 0.0281s/iter; left time: 157.7647s\n",
      "\t iters: 70 / 70, epoch: 20 | loss: -4.7560844\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00120937 0.02778532 0.00063255]\n",
      "\t speed: 0.0291s/iter; left time: 163.0941s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 20 Time: 2.2368571758270264 Steps: 70\n",
      "Train Loss: -4.7405819 | Val Loss: -4.8025106\n",
      "\t iters: 10 / 70, epoch: 21 | loss: -4.8551207\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00883123 0.0073349  0.01180542]\n",
      "\t speed: 0.0274s/iter; left time: 153.3798s\n",
      "\t iters: 20 / 70, epoch: 21 | loss: -4.3836651\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00103963 0.0044755  0.00187011]\n",
      "\t speed: 0.0281s/iter; left time: 156.7219s\n",
      "\t iters: 30 / 70, epoch: 21 | loss: -4.8255463\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05770908 0.00503021 0.01830575]\n",
      "\t speed: 0.0297s/iter; left time: 165.3840s\n",
      "\t iters: 40 / 70, epoch: 21 | loss: -4.8010092\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00620583 0.01656779 0.02396425]\n",
      "\t speed: 0.0286s/iter; left time: 159.1167s\n",
      "\t iters: 50 / 70, epoch: 21 | loss: -4.9775457\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00855968 0.01325775 0.03933565]\n",
      "\t speed: 0.0284s/iter; left time: 157.4824s\n",
      "\t iters: 60 / 70, epoch: 21 | loss: -4.7200136\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04983316 0.03837917 0.00281612]\n",
      "\t speed: 0.0279s/iter; left time: 154.5522s\n",
      "\t iters: 70 / 70, epoch: 21 | loss: -4.5669522\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01522788 0.0199458  0.02723195]\n",
      "\t speed: 0.0282s/iter; left time: 156.1501s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 21 Time: 2.03369402885437 Steps: 70\n",
      "Train Loss: -4.7860045 | Val Loss: -4.7601530\n",
      "\t iters: 10 / 70, epoch: 22 | loss: -4.8109846\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01386413 0.01814305 0.02190319]\n",
      "\t speed: 0.0360s/iter; left time: 198.7882s\n",
      "\t iters: 20 / 70, epoch: 22 | loss: -4.7388134\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00334716 0.01775843 0.01712738]\n",
      "\t speed: 0.0280s/iter; left time: 154.5203s\n",
      "\t iters: 30 / 70, epoch: 22 | loss: -4.3249369\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04434303 0.00830724 0.01143102]\n",
      "\t speed: 0.0280s/iter; left time: 154.2570s\n",
      "\t iters: 40 / 70, epoch: 22 | loss: -4.5496750\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00179055 0.0001973  0.00144705]\n",
      "\t speed: 0.0283s/iter; left time: 155.1691s\n",
      "\t iters: 50 / 70, epoch: 22 | loss: -4.6594381\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0149599  0.0039633  0.00329484]\n",
      "\t speed: 0.0287s/iter; left time: 157.3210s\n",
      "\t iters: 60 / 70, epoch: 22 | loss: -4.7551231\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05369789 0.03201349 0.02020312]\n",
      "\t speed: 0.0285s/iter; left time: 155.8294s\n",
      "\t iters: 70 / 70, epoch: 22 | loss: -4.9791374\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03973562 0.02139317 0.01424086]\n",
      "\t speed: 0.0290s/iter; left time: 158.3756s\n",
      "Validation loss descreased: -4.843667507171631 -> -4.905841827392578\n",
      "Epoch: 22 Time: 2.142082929611206 Steps: 70\n",
      "Train Loss: -4.8048682 | Val Loss: -4.9058418\n",
      "\t iters: 10 / 70, epoch: 23 | loss: -4.8414021\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04484784 0.00977241 0.03195387]\n",
      "\t speed: 0.0279s/iter; left time: 152.1132s\n",
      "\t iters: 20 / 70, epoch: 23 | loss: -4.8249311\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0003016  0.00179308 0.00086959]\n",
      "\t speed: 0.0290s/iter; left time: 157.5764s\n",
      "\t iters: 30 / 70, epoch: 23 | loss: -4.6398120\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00051593 0.00035917 0.00562981]\n",
      "\t speed: 0.0282s/iter; left time: 153.1352s\n",
      "\t iters: 40 / 70, epoch: 23 | loss: -4.8021493\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01301881 0.01638558 0.01491821]\n",
      "\t speed: 0.0287s/iter; left time: 155.4335s\n",
      "\t iters: 50 / 70, epoch: 23 | loss: -4.6038599\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01223591 0.04778237 0.03484496]\n",
      "\t speed: 0.0288s/iter; left time: 155.8013s\n",
      "\t iters: 60 / 70, epoch: 23 | loss: -4.8751364\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04076476 0.00932467 0.02499263]\n",
      "\t speed: 0.0388s/iter; left time: 209.5690s\n",
      "\t iters: 70 / 70, epoch: 23 | loss: -4.5112820\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00054336 0.00061023 0.00050994]\n",
      "\t speed: 0.0284s/iter; left time: 152.9537s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 23 Time: 2.1508688926696777 Steps: 70\n",
      "Train Loss: -4.8291225 | Val Loss: -4.6084913\n",
      "\t iters: 10 / 70, epoch: 24 | loss: -4.9253178\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00331923 0.01686828 0.00214935]\n",
      "\t speed: 0.0278s/iter; left time: 149.4992s\n",
      "\t iters: 20 / 70, epoch: 24 | loss: -4.9633641\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00652183 0.01347905 0.00292327]\n",
      "\t speed: 0.0286s/iter; left time: 153.8575s\n",
      "\t iters: 30 / 70, epoch: 24 | loss: -4.8628674\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01991171 0.01042648 0.00361937]\n",
      "\t speed: 0.0287s/iter; left time: 153.8743s\n",
      "\t iters: 40 / 70, epoch: 24 | loss: -4.8349433\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00779928 0.03652041 0.05118583]\n",
      "\t speed: 0.0270s/iter; left time: 144.7024s\n",
      "\t iters: 50 / 70, epoch: 24 | loss: -5.2471676\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00448287 0.0049718  0.00376878]\n",
      "\t speed: 0.0274s/iter; left time: 146.3635s\n",
      "\t iters: 60 / 70, epoch: 24 | loss: -5.0054073\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01963977 0.01992516 0.01683725]\n",
      "\t speed: 0.0285s/iter; left time: 151.6863s\n",
      "\t iters: 70 / 70, epoch: 24 | loss: -5.1824322\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02459549 0.00327338 0.03244101]\n",
      "\t speed: 0.0282s/iter; left time: 150.2567s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 24 Time: 2.013162851333618 Steps: 70\n",
      "Train Loss: -4.8685191 | Val Loss: -4.7587988\n",
      "\t iters: 10 / 70, epoch: 25 | loss: -5.1319084\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00182831 0.00036851 0.00584062]\n",
      "\t speed: 0.0268s/iter; left time: 142.4126s\n",
      "\t iters: 20 / 70, epoch: 25 | loss: -5.0670114\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02223049 0.00232098 0.00127968]\n",
      "\t speed: 0.0280s/iter; left time: 148.6663s\n",
      "\t iters: 30 / 70, epoch: 25 | loss: -5.1468444\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.05522322 0.0284644  0.0394638 ]\n",
      "\t speed: 0.0280s/iter; left time: 147.9821s\n",
      "\t iters: 40 / 70, epoch: 25 | loss: -4.8765278\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04157333 0.01545731 0.03623978]\n",
      "\t speed: 0.0277s/iter; left time: 146.2691s\n",
      "\t iters: 50 / 70, epoch: 25 | loss: -4.9198866\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00649594 0.00330836 0.01251152]\n",
      "\t speed: 0.0375s/iter; left time: 197.6768s\n",
      "\t iters: 60 / 70, epoch: 25 | loss: -5.0435853\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0358213  0.00192853 0.03947926]\n",
      "\t speed: 0.0266s/iter; left time: 139.9928s\n",
      "\t iters: 70 / 70, epoch: 25 | loss: -4.8253407\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03440699 0.05767203 0.03601565]\n",
      "\t speed: 0.0271s/iter; left time: 142.2376s\n",
      "Validation loss descreased: -4.905841827392578 -> -4.986974875132243\n",
      "Epoch: 25 Time: 2.081770181655884 Steps: 70\n",
      "Train Loss: -4.8633386 | Val Loss: -4.9869749\n",
      "\t iters: 10 / 70, epoch: 26 | loss: -4.9972048\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03297272 0.00643829 0.00561009]\n",
      "\t speed: 0.0275s/iter; left time: 144.1490s\n",
      "\t iters: 20 / 70, epoch: 26 | loss: -4.7488489\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02501371 0.03627482 0.01610709]\n",
      "\t speed: 0.0287s/iter; left time: 150.1354s\n",
      "\t iters: 30 / 70, epoch: 26 | loss: -4.6616879\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00185072 0.00243793 0.00046579]\n",
      "\t speed: 0.0281s/iter; left time: 146.7366s\n",
      "\t iters: 40 / 70, epoch: 26 | loss: -5.1491370\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00909117 0.02336105 0.03393378]\n",
      "\t speed: 0.0280s/iter; left time: 145.9692s\n",
      "\t iters: 50 / 70, epoch: 26 | loss: -5.1714697\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0004065  0.00213482 0.00121173]\n",
      "\t speed: 0.0280s/iter; left time: 145.5722s\n",
      "\t iters: 60 / 70, epoch: 26 | loss: -4.5062256\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00065494 0.00045056 0.00033474]\n",
      "\t speed: 0.0273s/iter; left time: 141.7425s\n",
      "\t iters: 70 / 70, epoch: 26 | loss: -4.9391699\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02259054 0.00564886 0.00211119]\n",
      "\t speed: 0.0279s/iter; left time: 144.3538s\n",
      "Validation loss descreased: -4.986974875132243 -> -4.9870913823445635\n",
      "Epoch: 26 Time: 2.0189526081085205 Steps: 70\n",
      "Train Loss: -4.8816040 | Val Loss: -4.9870914\n",
      "\t iters: 10 / 70, epoch: 27 | loss: -5.0505090\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00024866 0.00032193 0.00417771]\n",
      "\t speed: 0.0273s/iter; left time: 140.9577s\n",
      "\t iters: 20 / 70, epoch: 27 | loss: -4.9221916\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02343639 0.0104324  0.03745697]\n",
      "\t speed: 0.0289s/iter; left time: 149.2970s\n",
      "\t iters: 30 / 70, epoch: 27 | loss: -4.8744917\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03469725 0.00078389 0.00156426]\n",
      "\t speed: 0.0390s/iter; left time: 201.1099s\n",
      "\t iters: 40 / 70, epoch: 27 | loss: -4.9742794\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00057066 0.00093991 0.00609832]\n",
      "\t speed: 0.0280s/iter; left time: 143.8246s\n",
      "\t iters: 50 / 70, epoch: 27 | loss: -5.2004294\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00415323 0.05019459 0.05019459]\n",
      "\t speed: 0.0284s/iter; left time: 145.5457s\n",
      "\t iters: 60 / 70, epoch: 27 | loss: -4.5213680\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01249593 0.0408324  0.02466247]\n",
      "\t speed: 0.0277s/iter; left time: 141.7254s\n",
      "\t iters: 70 / 70, epoch: 27 | loss: -5.0298576\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.042293   0.04245266 0.04847312]\n",
      "\t speed: 0.0284s/iter; left time: 144.9900s\n",
      "Validation loss descreased: -4.9870913823445635 -> -4.991617202758789\n",
      "Epoch: 27 Time: 2.1448657512664795 Steps: 70\n",
      "Train Loss: -4.9068353 | Val Loss: -4.9916172\n",
      "\t iters: 10 / 70, epoch: 28 | loss: -5.0199604\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00488634 0.01713843 0.00323783]\n",
      "\t speed: 0.0274s/iter; left time: 139.9038s\n",
      "\t iters: 20 / 70, epoch: 28 | loss: -4.9342933\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02709304 0.00265048 0.00168346]\n",
      "\t speed: 0.0287s/iter; left time: 146.0034s\n",
      "\t iters: 30 / 70, epoch: 28 | loss: -4.9312916\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00520675 0.00286451 0.01129714]\n",
      "\t speed: 0.0291s/iter; left time: 147.6089s\n",
      "\t iters: 40 / 70, epoch: 28 | loss: -4.9511704\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.004055   0.05349166 0.00849851]\n",
      "\t speed: 0.0286s/iter; left time: 145.2487s\n",
      "\t iters: 50 / 70, epoch: 28 | loss: -5.0714755\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00274015 0.01043472 0.00365022]\n",
      "\t speed: 0.0283s/iter; left time: 143.2730s\n",
      "\t iters: 60 / 70, epoch: 28 | loss: -4.7824659\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01188026 0.012968   0.01935166]\n",
      "\t speed: 0.0290s/iter; left time: 146.5558s\n",
      "\t iters: 70 / 70, epoch: 28 | loss: -4.8835893\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02837938 0.04980132 0.03367173]\n",
      "\t speed: 0.0288s/iter; left time: 145.1193s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 28 Time: 2.053510904312134 Steps: 70\n",
      "Train Loss: -4.9334498 | Val Loss: -4.8868491\n",
      "\t iters: 10 / 70, epoch: 29 | loss: -4.9180374\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00166311 0.00122094 0.00085713]\n",
      "\t speed: 0.0401s/iter; left time: 201.5860s\n",
      "\t iters: 20 / 70, epoch: 29 | loss: -5.0485773\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03750525 0.0084321  0.03591547]\n",
      "\t speed: 0.0305s/iter; left time: 153.1805s\n",
      "\t iters: 30 / 70, epoch: 29 | loss: -5.0685120\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.011138   0.0125705  0.00038548]\n",
      "\t speed: 0.0303s/iter; left time: 152.0048s\n",
      "\t iters: 40 / 70, epoch: 29 | loss: -4.8642883\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01258767 0.04718937 0.01940121]\n",
      "\t speed: 0.0287s/iter; left time: 143.3734s\n",
      "\t iters: 50 / 70, epoch: 29 | loss: -5.0179243\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03826864 0.00794248 0.00151161]\n",
      "\t speed: 0.0289s/iter; left time: 144.3907s\n",
      "\t iters: 60 / 70, epoch: 29 | loss: -5.0944672\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01677808 0.04863497 0.00058692]\n",
      "\t speed: 0.0301s/iter; left time: 149.9675s\n",
      "\t iters: 70 / 70, epoch: 29 | loss: -4.9214697\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03251226 0.01312293 0.04868679]\n",
      "\t speed: 0.0289s/iter; left time: 143.6962s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 29 Time: 2.2294774055480957 Steps: 70\n",
      "Train Loss: -4.9303579 | Val Loss: -4.9596601\n",
      "\t iters: 10 / 70, epoch: 30 | loss: -4.7980967\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01058551 0.0287086  0.04855847]\n",
      "\t speed: 0.0277s/iter; left time: 137.4711s\n",
      "\t iters: 20 / 70, epoch: 30 | loss: -4.9912286\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0015989  0.00410841 0.00391612]\n",
      "\t speed: 0.0284s/iter; left time: 140.6298s\n",
      "\t iters: 30 / 70, epoch: 30 | loss: -4.9415717\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02019323 0.00880131 0.02025179]\n",
      "\t speed: 0.0288s/iter; left time: 142.2899s\n",
      "\t iters: 40 / 70, epoch: 30 | loss: -5.0918427\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00820587 0.00339953 0.01134584]\n",
      "\t speed: 0.0286s/iter; left time: 141.2031s\n",
      "\t iters: 50 / 70, epoch: 30 | loss: -4.5364695\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00255762 0.00136883 0.04372719]\n",
      "\t speed: 0.0299s/iter; left time: 147.1336s\n",
      "\t iters: 60 / 70, epoch: 30 | loss: -5.1066790\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00416596 0.00731118 0.00411363]\n",
      "\t speed: 0.0319s/iter; left time: 156.8023s\n",
      "\t iters: 70 / 70, epoch: 30 | loss: -5.0041409\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00180071 0.00104173 0.00092252]\n",
      "\t speed: 0.0386s/iter; left time: 189.3176s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 30 Time: 2.19439697265625 Steps: 70\n",
      "Train Loss: -4.9390773 | Val Loss: -4.9028552\n",
      "\t iters: 10 / 70, epoch: 31 | loss: -4.9233999\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00568622 0.0366246  0.00579021]\n",
      "\t speed: 0.0289s/iter; left time: 141.5882s\n",
      "\t iters: 20 / 70, epoch: 31 | loss: -4.8675232\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00131812 0.00106192 0.00312817]\n",
      "\t speed: 0.0294s/iter; left time: 143.5240s\n",
      "\t iters: 30 / 70, epoch: 31 | loss: -5.1425333\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00911138 0.02253579 0.0079297 ]\n",
      "\t speed: 0.0309s/iter; left time: 150.4837s\n",
      "\t iters: 40 / 70, epoch: 31 | loss: -4.9291029\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00540504 0.00845828 0.00701914]\n",
      "\t speed: 0.0293s/iter; left time: 142.2525s\n",
      "\t iters: 50 / 70, epoch: 31 | loss: -4.6218295\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01961785 0.01521506 0.01781379]\n",
      "\t speed: 0.0284s/iter; left time: 137.8177s\n",
      "\t iters: 60 / 70, epoch: 31 | loss: -5.0400877\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03510255 0.04873257 0.00965914]\n",
      "\t speed: 0.0310s/iter; left time: 149.9596s\n",
      "\t iters: 70 / 70, epoch: 31 | loss: -5.0255609\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02065183 0.00521369 0.02236147]\n",
      "\t speed: 0.0287s/iter; left time: 138.5809s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 31 Time: 2.1172192096710205 Steps: 70\n",
      "Train Loss: -4.9738350 | Val Loss: -4.9408763\n",
      "\t iters: 10 / 70, epoch: 32 | loss: -4.8225107\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00828672 0.00600516 0.0060337 ]\n",
      "\t speed: 0.0298s/iter; left time: 143.6661s\n",
      "\t iters: 20 / 70, epoch: 32 | loss: -5.0115938\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00106736 0.00029492 0.00412321]\n",
      "\t speed: 0.0294s/iter; left time: 141.4672s\n",
      "\t iters: 30 / 70, epoch: 32 | loss: -5.3134503\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00252015 0.00121403 0.00334067]\n",
      "\t speed: 0.0292s/iter; left time: 140.4036s\n",
      "\t iters: 40 / 70, epoch: 32 | loss: -5.1699934\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00443906 0.00596507 0.0186741 ]\n",
      "\t speed: 0.0296s/iter; left time: 141.9745s\n",
      "\t iters: 50 / 70, epoch: 32 | loss: -4.6625690\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01370843 0.03794875 0.04200834]\n",
      "\t speed: 0.0382s/iter; left time: 182.4315s\n",
      "\t iters: 60 / 70, epoch: 32 | loss: -5.2262259\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00410791 0.00103975 0.00207999]\n",
      "\t speed: 0.0292s/iter; left time: 139.1614s\n",
      "\t iters: 70 / 70, epoch: 32 | loss: -4.7787495\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00136586 0.00039688 0.00207191]\n",
      "\t speed: 0.0293s/iter; left time: 139.3387s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 32 Time: 2.2014122009277344 Steps: 70\n",
      "Train Loss: -5.0061650 | Val Loss: -4.5025527\n",
      "\t iters: 10 / 70, epoch: 33 | loss: -5.1052032\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00030548 0.00115094 0.0009439 ]\n",
      "\t speed: 0.0285s/iter; left time: 135.3707s\n",
      "\t iters: 20 / 70, epoch: 33 | loss: -4.7333288\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0249367  0.04813011 0.0270414 ]\n",
      "\t speed: 0.0297s/iter; left time: 140.6942s\n",
      "\t iters: 30 / 70, epoch: 33 | loss: -5.2830811\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00216909 0.00634216 0.0026128 ]\n",
      "\t speed: 0.0284s/iter; left time: 134.2630s\n",
      "\t iters: 40 / 70, epoch: 33 | loss: -4.9377594\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00099022 0.00461635 0.0059552 ]\n",
      "\t speed: 0.0286s/iter; left time: 135.0974s\n",
      "\t iters: 50 / 70, epoch: 33 | loss: -4.7007694\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00362284 0.0090606  0.00286982]\n",
      "\t speed: 0.0297s/iter; left time: 139.9824s\n",
      "\t iters: 60 / 70, epoch: 33 | loss: -4.7532787\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0275133  0.03962609 0.03318221]\n",
      "\t speed: 0.0290s/iter; left time: 136.4973s\n",
      "\t iters: 70 / 70, epoch: 33 | loss: -4.8818107\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01165631 0.00413041 0.00370803]\n",
      "\t speed: 0.0307s/iter; left time: 144.1121s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 33 Time: 2.1060478687286377 Steps: 70\n",
      "Train Loss: -4.9847689 | Val Loss: -4.8548282\n",
      "\t iters: 10 / 70, epoch: 34 | loss: -5.0793834\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01943159 0.00827023 0.00961222]\n",
      "\t speed: 0.0288s/iter; left time: 135.0228s\n",
      "\t iters: 20 / 70, epoch: 34 | loss: -4.9724650\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01745748 0.0256095  0.04755073]\n",
      "\t speed: 0.0298s/iter; left time: 139.2587s\n",
      "\t iters: 30 / 70, epoch: 34 | loss: -4.9815140\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00085804 0.02195786 0.00078236]\n",
      "\t speed: 0.0374s/iter; left time: 174.5505s\n",
      "\t iters: 40 / 70, epoch: 34 | loss: -4.9514084\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01941429 0.04803547 0.02171119]\n",
      "\t speed: 0.0309s/iter; left time: 143.6037s\n",
      "\t iters: 50 / 70, epoch: 34 | loss: -5.0578623\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00114753 0.00252741 0.01396088]\n",
      "\t speed: 0.0299s/iter; left time: 138.5973s\n",
      "\t iters: 60 / 70, epoch: 34 | loss: -5.1302681\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03814179 0.03350725 0.02186671]\n",
      "\t speed: 0.0300s/iter; left time: 138.8124s\n",
      "\t iters: 70 / 70, epoch: 34 | loss: -5.0978494\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01164505 0.00323561 0.01223836]\n",
      "\t speed: 0.0302s/iter; left time: 139.4138s\n",
      "Validation loss descreased: -4.991617202758789 -> -5.043075879414876\n",
      "Epoch: 34 Time: 2.2463185787200928 Steps: 70\n",
      "Train Loss: -4.9961986 | Val Loss: -5.0430759\n",
      "\t iters: 10 / 70, epoch: 35 | loss: -5.2558489\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00534491 0.00854532 0.01846833]\n",
      "\t speed: 0.0289s/iter; left time: 133.2867s\n",
      "\t iters: 20 / 70, epoch: 35 | loss: -5.0076785\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00128595 0.0006531  0.00041804]\n",
      "\t speed: 0.0302s/iter; left time: 139.1494s\n",
      "\t iters: 30 / 70, epoch: 35 | loss: -4.8628111\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00995113 0.00058    0.01356054]\n",
      "\t speed: 0.0303s/iter; left time: 138.8845s\n",
      "\t iters: 40 / 70, epoch: 35 | loss: -4.6007471\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02613074 0.01571702 0.03186993]\n",
      "\t speed: 0.0305s/iter; left time: 139.5746s\n",
      "\t iters: 50 / 70, epoch: 35 | loss: -5.1074657\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02371671 0.00404936 0.00779866]\n",
      "\t speed: 0.0312s/iter; left time: 142.4722s\n",
      "\t iters: 60 / 70, epoch: 35 | loss: -5.0732765\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01488288 0.00489007 0.0077819 ]\n",
      "\t speed: 0.0284s/iter; left time: 129.5122s\n",
      "\t iters: 70 / 70, epoch: 35 | loss: -4.8646870\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00463244 0.01045456 0.00449864]\n",
      "\t speed: 0.0297s/iter; left time: 135.1082s\n",
      "Validation loss descreased: -5.043075879414876 -> -5.044336318969727\n",
      "Epoch: 35 Time: 2.1566781997680664 Steps: 70\n",
      "Train Loss: -5.0050309 | Val Loss: -5.0443363\n",
      "\t iters: 10 / 70, epoch: 36 | loss: -4.9451666\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00377557 0.00747042 0.00386687]\n",
      "\t speed: 0.0283s/iter; left time: 128.6518s\n",
      "\t iters: 20 / 70, epoch: 36 | loss: -5.1262565\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00309113 0.00539289 0.00243175]\n",
      "\t speed: 0.0395s/iter; left time: 179.0588s\n",
      "\t iters: 30 / 70, epoch: 36 | loss: -5.0593033\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00310861 0.01806002 0.02511537]\n",
      "\t speed: 0.0284s/iter; left time: 128.4173s\n",
      "\t iters: 40 / 70, epoch: 36 | loss: -4.8193660\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02912247 0.00987673 0.0114463 ]\n",
      "\t speed: 0.0281s/iter; left time: 126.7750s\n",
      "\t iters: 50 / 70, epoch: 36 | loss: -4.9264240\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00643186 0.00070314 0.00166194]\n",
      "\t speed: 0.0296s/iter; left time: 133.0170s\n",
      "\t iters: 60 / 70, epoch: 36 | loss: -5.0211706\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00505915 0.00520826 0.00094756]\n",
      "\t speed: 0.0284s/iter; left time: 127.6507s\n",
      "\t iters: 70 / 70, epoch: 36 | loss: -4.8074622\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0095093  0.00750416 0.02180387]\n",
      "\t speed: 0.0313s/iter; left time: 140.3357s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 36 Time: 2.196615695953369 Steps: 70\n",
      "Train Loss: -4.9952472 | Val Loss: -5.0175085\n",
      "\t iters: 10 / 70, epoch: 37 | loss: -4.9241734\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04760825 0.0025359  0.01962492]\n",
      "\t speed: 0.0287s/iter; left time: 128.1996s\n",
      "\t iters: 20 / 70, epoch: 37 | loss: -4.9698896\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.009139   0.00197251 0.0075428 ]\n",
      "\t speed: 0.0296s/iter; left time: 131.8552s\n",
      "\t iters: 30 / 70, epoch: 37 | loss: -5.1450524\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01699126 0.01769462 0.03313966]\n",
      "\t speed: 0.0289s/iter; left time: 128.6695s\n",
      "\t iters: 40 / 70, epoch: 37 | loss: -4.9083800\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03576728 0.00356543 0.01642089]\n",
      "\t speed: 0.0291s/iter; left time: 129.2723s\n",
      "\t iters: 50 / 70, epoch: 37 | loss: -5.0054951\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0027166  0.0020221  0.03809613]\n",
      "\t speed: 0.0287s/iter; left time: 127.3365s\n",
      "\t iters: 60 / 70, epoch: 37 | loss: -5.0569654\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00978354 0.04290159 0.01575848]\n",
      "\t speed: 0.0287s/iter; left time: 127.0365s\n",
      "\t iters: 70 / 70, epoch: 37 | loss: -4.8645530\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00894492 0.00176629 0.03428997]\n",
      "\t speed: 0.0394s/iter; left time: 173.8566s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 37 Time: 2.1820015907287598 Steps: 70\n",
      "Train Loss: -5.0110735 | Val Loss: -4.9910955\n",
      "\t iters: 10 / 70, epoch: 38 | loss: -4.9134521\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01752705 0.02246842 0.01358928]\n",
      "\t speed: 0.0267s/iter; left time: 117.2951s\n",
      "\t iters: 20 / 70, epoch: 38 | loss: -5.0889349\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0037558  0.01957987 0.00859599]\n",
      "\t speed: 0.0273s/iter; left time: 119.8400s\n",
      "\t iters: 30 / 70, epoch: 38 | loss: -4.9146643\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00314027 0.00093513 0.00101391]\n",
      "\t speed: 0.0295s/iter; left time: 129.0923s\n",
      "\t iters: 40 / 70, epoch: 38 | loss: -5.0445089\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01431923 0.01393444 0.0092754 ]\n",
      "\t speed: 0.0303s/iter; left time: 132.4161s\n",
      "\t iters: 50 / 70, epoch: 38 | loss: -4.9488683\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04164699 0.0473009  0.03951219]\n",
      "\t speed: 0.0282s/iter; left time: 122.8440s\n",
      "\t iters: 60 / 70, epoch: 38 | loss: -4.9909363\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03023995 0.02640137 0.0071413 ]\n",
      "\t speed: 0.0287s/iter; left time: 125.0853s\n",
      "\t iters: 70 / 70, epoch: 38 | loss: -5.0563722\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01249088 0.03777504 0.04789189]\n",
      "\t speed: 0.0285s/iter; left time: 123.8797s\n",
      "Validation loss descreased: -5.044336318969727 -> -5.112780570983887\n",
      "Epoch: 38 Time: 2.062065601348877 Steps: 70\n",
      "Train Loss: -5.0414315 | Val Loss: -5.1127806\n",
      "\t iters: 10 / 70, epoch: 39 | loss: -5.1389756\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01861502 0.04496542 0.02352978]\n",
      "\t speed: 0.0266s/iter; left time: 115.0964s\n",
      "\t iters: 20 / 70, epoch: 39 | loss: -4.9563751\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04583481 0.01906081 0.02783739]\n",
      "\t speed: 0.0250s/iter; left time: 107.9256s\n",
      "\t iters: 30 / 70, epoch: 39 | loss: -4.8732557\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00611608 0.00364038 0.04757523]\n",
      "\t speed: 0.0282s/iter; left time: 121.5001s\n",
      "\t iters: 40 / 70, epoch: 39 | loss: -4.6745901\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00189285 0.01098325 0.00247342]\n",
      "\t speed: 0.0284s/iter; left time: 122.0200s\n",
      "\t iters: 50 / 70, epoch: 39 | loss: -4.9796267\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0041419  0.00049194 0.0043825 ]\n",
      "\t speed: 0.0294s/iter; left time: 125.9777s\n",
      "\t iters: 60 / 70, epoch: 39 | loss: -5.0797119\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01845404 0.02295935 0.03736895]\n",
      "\t speed: 0.0386s/iter; left time: 165.2381s\n",
      "\t iters: 70 / 70, epoch: 39 | loss: -5.1434073\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00692815 0.00064348 0.00067744]\n",
      "\t speed: 0.0282s/iter; left time: 120.6428s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 39 Time: 2.094841957092285 Steps: 70\n",
      "Train Loss: -5.0200465 | Val Loss: -4.9743147\n",
      "\t iters: 10 / 70, epoch: 40 | loss: -4.8432961\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02132049 0.03286691 0.02929503]\n",
      "\t speed: 0.0257s/iter; left time: 109.4685s\n",
      "\t iters: 20 / 70, epoch: 40 | loss: -5.1382303\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0471299  0.01504375 0.04450698]\n",
      "\t speed: 0.0254s/iter; left time: 107.8173s\n",
      "\t iters: 30 / 70, epoch: 40 | loss: -4.9869251\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00966932 0.01484837 0.02965037]\n",
      "\t speed: 0.0288s/iter; left time: 122.0044s\n",
      "\t iters: 40 / 70, epoch: 40 | loss: -4.9899197\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00405968 0.00105301 0.001338  ]\n",
      "\t speed: 0.0282s/iter; left time: 119.2475s\n",
      "\t iters: 50 / 70, epoch: 40 | loss: -4.9167514\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01699584 0.04334079 0.04414252]\n",
      "\t speed: 0.0284s/iter; left time: 119.9183s\n",
      "\t iters: 60 / 70, epoch: 40 | loss: -4.5087409\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00546046 0.03883946 0.03779233]\n",
      "\t speed: 0.0280s/iter; left time: 117.7789s\n",
      "\t iters: 70 / 70, epoch: 40 | loss: -4.6634402\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00996249 0.00579716 0.00538437]\n",
      "\t speed: 0.0283s/iter; left time: 118.6893s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 40 Time: 1.9767060279846191 Steps: 70\n",
      "Train Loss: -5.0353371 | Val Loss: -4.8323989\n",
      "\t iters: 10 / 70, epoch: 41 | loss: -4.7550325\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02104678 0.00779606 0.02448079]\n",
      "\t speed: 0.0269s/iter; left time: 112.6407s\n",
      "\t iters: 20 / 70, epoch: 41 | loss: -4.7473302\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00337968 0.00330699 0.00556255]\n",
      "\t speed: 0.0279s/iter; left time: 116.8292s\n",
      "\t iters: 30 / 70, epoch: 41 | loss: -5.1496353\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00184887 0.00049117 0.0007766 ]\n",
      "\t speed: 0.0284s/iter; left time: 118.6248s\n",
      "\t iters: 40 / 70, epoch: 41 | loss: -5.0840359\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02408921 0.00545878 0.01681948]\n",
      "\t speed: 0.0380s/iter; left time: 158.0427s\n",
      "\t iters: 50 / 70, epoch: 41 | loss: -5.1286564\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.014715   0.04739008 0.02398769]\n",
      "\t speed: 0.0298s/iter; left time: 123.7727s\n",
      "\t iters: 60 / 70, epoch: 41 | loss: -4.8689775\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00422925 0.00150532 0.0329343 ]\n",
      "\t speed: 0.0299s/iter; left time: 123.9278s\n",
      "\t iters: 70 / 70, epoch: 41 | loss: -5.2127309\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01604259 0.0166896  0.01009461]\n",
      "\t speed: 0.0288s/iter; left time: 118.8730s\n",
      "Validation loss descreased: -5.112780570983887 -> -5.143496831258138\n",
      "Epoch: 41 Time: 2.1678550243377686 Steps: 70\n",
      "Train Loss: -5.0305390 | Val Loss: -5.1434968\n",
      "\t iters: 10 / 70, epoch: 42 | loss: -5.0011673\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00667684 0.02111697 0.00480991]\n",
      "\t speed: 0.0279s/iter; left time: 114.7833s\n",
      "\t iters: 20 / 70, epoch: 42 | loss: -5.2008629\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01870723 0.01271756 0.03140851]\n",
      "\t speed: 0.0269s/iter; left time: 110.6483s\n",
      "\t iters: 30 / 70, epoch: 42 | loss: -5.0146236\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01415247 0.01764123 0.02592683]\n",
      "\t speed: 0.0280s/iter; left time: 114.8005s\n",
      "\t iters: 40 / 70, epoch: 42 | loss: -5.2465963\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02053753 0.03756294 0.01313719]\n",
      "\t speed: 0.0283s/iter; left time: 115.5918s\n",
      "\t iters: 50 / 70, epoch: 42 | loss: -5.0383997\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01975469 0.02937332 0.01396315]\n",
      "\t speed: 0.0274s/iter; left time: 111.9812s\n",
      "\t iters: 60 / 70, epoch: 42 | loss: -4.9590950\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0004654  0.01567585 0.00257022]\n",
      "\t speed: 0.0284s/iter; left time: 115.5997s\n",
      "\t iters: 70 / 70, epoch: 42 | loss: -4.8974495\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00024511 0.013977   0.00233907]\n",
      "\t speed: 0.0304s/iter; left time: 123.4577s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 42 Time: 2.0280416011810303 Steps: 70\n",
      "Train Loss: -5.0749541 | Val Loss: -5.0846020\n",
      "\t iters: 10 / 70, epoch: 43 | loss: -5.2860570\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00422619 0.00092324 0.00078051]\n",
      "\t speed: 0.0269s/iter; left time: 109.1224s\n",
      "\t iters: 20 / 70, epoch: 43 | loss: -5.3667383\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00310259 0.00119486 0.00044685]\n",
      "\t speed: 0.0351s/iter; left time: 141.8190s\n",
      "\t iters: 30 / 70, epoch: 43 | loss: -4.9479513\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00075753 0.00128144 0.01673041]\n",
      "\t speed: 0.0275s/iter; left time: 110.6794s\n",
      "\t iters: 40 / 70, epoch: 43 | loss: -5.1377506\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03843526 0.02291184 0.02012794]\n",
      "\t speed: 0.0269s/iter; left time: 108.0446s\n",
      "\t iters: 50 / 70, epoch: 43 | loss: -5.0854125\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03072315 0.04923671 0.02825178]\n",
      "\t speed: 0.0271s/iter; left time: 108.8145s\n",
      "\t iters: 60 / 70, epoch: 43 | loss: -4.9892182\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00278373 0.00055293 0.00108253]\n",
      "\t speed: 0.0276s/iter; left time: 110.4553s\n",
      "\t iters: 70 / 70, epoch: 43 | loss: -5.2336702\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02598356 0.03983911 0.0335696 ]\n",
      "\t speed: 0.0267s/iter; left time: 106.6318s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 43 Time: 2.0295512676239014 Steps: 70\n",
      "Train Loss: -5.0670512 | Val Loss: -5.0674979\n",
      "\t iters: 10 / 70, epoch: 44 | loss: -5.1437798\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00228343 0.00269068 0.00289057]\n",
      "\t speed: 0.0263s/iter; left time: 104.6359s\n",
      "\t iters: 20 / 70, epoch: 44 | loss: -5.1234107\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00166088 0.00154306 0.00073483]\n",
      "\t speed: 0.0273s/iter; left time: 108.4122s\n",
      "\t iters: 30 / 70, epoch: 44 | loss: -4.8152132\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01809784 0.01265993 0.01741828]\n",
      "\t speed: 0.0288s/iter; left time: 114.1783s\n",
      "\t iters: 40 / 70, epoch: 44 | loss: -5.0253258\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01237089 0.01340303 0.01571785]\n",
      "\t speed: 0.0276s/iter; left time: 108.9507s\n",
      "\t iters: 50 / 70, epoch: 44 | loss: -5.1393681\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00358352 0.00485439 0.00370503]\n",
      "\t speed: 0.0289s/iter; left time: 113.8723s\n",
      "\t iters: 60 / 70, epoch: 44 | loss: -4.9612112\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02069782 0.01833296 0.03625823]\n",
      "\t speed: 0.0289s/iter; left time: 113.6436s\n",
      "\t iters: 70 / 70, epoch: 44 | loss: -5.2098427\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00091761 0.01152278 0.02507692]\n",
      "\t speed: 0.0292s/iter; left time: 114.6033s\n",
      "Validation loss descreased: -5.143496831258138 -> -5.313997745513916\n",
      "Epoch: 44 Time: 2.0403997898101807 Steps: 70\n",
      "Train Loss: -5.0851093 | Val Loss: -5.3139977\n",
      "\t iters: 10 / 70, epoch: 45 | loss: -4.6949487\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00539617 0.00127534 0.00305817]\n",
      "\t speed: 0.0379s/iter; left time: 148.2596s\n",
      "\t iters: 20 / 70, epoch: 45 | loss: -4.8478260\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0212823  0.0146495  0.01547004]\n",
      "\t speed: 0.0286s/iter; left time: 111.7405s\n",
      "\t iters: 30 / 70, epoch: 45 | loss: -5.2719107\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00968993 0.00314873 0.02434498]\n",
      "\t speed: 0.0289s/iter; left time: 112.3130s\n",
      "\t iters: 40 / 70, epoch: 45 | loss: -4.8821745\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00151586 0.00094867 0.00403905]\n",
      "\t speed: 0.0287s/iter; left time: 111.3312s\n",
      "\t iters: 50 / 70, epoch: 45 | loss: -5.1919127\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00471727 0.00310507 0.01026611]\n",
      "\t speed: 0.0272s/iter; left time: 105.2235s\n",
      "\t iters: 60 / 70, epoch: 45 | loss: -5.0515366\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02274423 0.03859815 0.02628515]\n",
      "\t speed: 0.0271s/iter; left time: 104.6553s\n",
      "\t iters: 70 / 70, epoch: 45 | loss: -4.9185591\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0366854  0.00993754 0.01796935]\n",
      "\t speed: 0.0273s/iter; left time: 105.1946s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 45 Time: 2.1079397201538086 Steps: 70\n",
      "Train Loss: -5.0819768 | Val Loss: -5.0970084\n",
      "\t iters: 10 / 70, epoch: 46 | loss: -5.1866455\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00151347 0.01912707 0.00394032]\n",
      "\t speed: 0.0265s/iter; left time: 101.6083s\n",
      "\t iters: 20 / 70, epoch: 46 | loss: -5.1013947\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00057665 0.00200269 0.00031515]\n",
      "\t speed: 0.0273s/iter; left time: 104.5751s\n",
      "\t iters: 30 / 70, epoch: 46 | loss: -5.0827947\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00254749 0.0010416  0.00306868]\n",
      "\t speed: 0.0269s/iter; left time: 102.6771s\n",
      "\t iters: 40 / 70, epoch: 46 | loss: -5.2783937\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03816571 0.03449424 0.03556023]\n",
      "\t speed: 0.0276s/iter; left time: 105.1462s\n",
      "\t iters: 50 / 70, epoch: 46 | loss: -5.0322013\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01746311 0.00930835 0.02673148]\n",
      "\t speed: 0.0273s/iter; left time: 103.9287s\n",
      "\t iters: 60 / 70, epoch: 46 | loss: -4.8618245\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04136678 0.02721771 0.02611993]\n",
      "\t speed: 0.0382s/iter; left time: 144.8569s\n",
      "\t iters: 70 / 70, epoch: 46 | loss: -5.2795229\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00133484 0.00230534 0.00059666]\n",
      "\t speed: 0.0282s/iter; left time: 106.5269s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 46 Time: 2.0727155208587646 Steps: 70\n",
      "Train Loss: -5.0720323 | Val Loss: -5.0843101\n",
      "\t iters: 10 / 70, epoch: 47 | loss: -5.1073360\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00137434 0.0114297  0.00303661]\n",
      "\t speed: 0.0274s/iter; left time: 103.4353s\n",
      "\t iters: 20 / 70, epoch: 47 | loss: -5.0087328\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00054364 0.00082896 0.00152446]\n",
      "\t speed: 0.0286s/iter; left time: 107.5226s\n",
      "\t iters: 30 / 70, epoch: 47 | loss: -5.4885511\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00564969 0.00272612 0.01111701]\n",
      "\t speed: 0.0285s/iter; left time: 106.9565s\n",
      "\t iters: 40 / 70, epoch: 47 | loss: -5.0555344\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00206634 0.00288183 0.00272981]\n",
      "\t speed: 0.0269s/iter; left time: 100.7890s\n",
      "\t iters: 50 / 70, epoch: 47 | loss: -5.2937422\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04209523 0.04812961 0.03148308]\n",
      "\t speed: 0.0269s/iter; left time: 100.3596s\n",
      "\t iters: 60 / 70, epoch: 47 | loss: -5.0603733\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00727258 0.00061367 0.00642705]\n",
      "\t speed: 0.0274s/iter; left time: 102.1124s\n",
      "\t iters: 70 / 70, epoch: 47 | loss: -5.0587168\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02260866 0.0440854  0.01329686]\n",
      "\t speed: 0.0272s/iter; left time: 101.0657s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 47 Time: 1.9825608730316162 Steps: 70\n",
      "Train Loss: -5.1000525 | Val Loss: -5.1247802\n",
      "\t iters: 10 / 70, epoch: 48 | loss: -5.2665572\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02379428 0.005098   0.01219418]\n",
      "\t speed: 0.0257s/iter; left time: 94.9734s\n",
      "\t iters: 20 / 70, epoch: 48 | loss: -5.1459947\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00100247 0.00671733 0.00045467]\n",
      "\t speed: 0.0272s/iter; left time: 100.2987s\n",
      "\t iters: 30 / 70, epoch: 48 | loss: -5.0358782\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00561626 0.00241303 0.00262458]\n",
      "\t speed: 0.0269s/iter; left time: 99.1605s\n",
      "\t iters: 40 / 70, epoch: 48 | loss: -5.1430912\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0066026  0.01843325 0.0034057 ]\n",
      "\t speed: 0.0360s/iter; left time: 132.1224s\n",
      "\t iters: 50 / 70, epoch: 48 | loss: -5.0607271\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00264692 0.00381697 0.0042319 ]\n",
      "\t speed: 0.0272s/iter; left time: 99.6207s\n",
      "\t iters: 60 / 70, epoch: 48 | loss: -5.0922785\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00280461 0.00310277 0.00748753]\n",
      "\t speed: 0.0271s/iter; left time: 98.7919s\n",
      "\t iters: 70 / 70, epoch: 48 | loss: -4.7789865\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0005172  0.00237608 0.0083842 ]\n",
      "\t speed: 0.0270s/iter; left time: 98.2634s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 48 Time: 2.0249247550964355 Steps: 70\n",
      "Train Loss: -5.0973206 | Val Loss: -4.9655193\n",
      "\t iters: 10 / 70, epoch: 49 | loss: -5.0086670\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00659325 0.00082348 0.00170385]\n",
      "\t speed: 0.0274s/iter; left time: 99.6161s\n",
      "\t iters: 20 / 70, epoch: 49 | loss: -5.2808342\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01397632 0.03389661 0.02377369]\n",
      "\t speed: 0.0287s/iter; left time: 103.9620s\n",
      "\t iters: 30 / 70, epoch: 49 | loss: -5.1805897\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01383848 0.01855116 0.02057376]\n",
      "\t speed: 0.0285s/iter; left time: 103.0508s\n",
      "\t iters: 40 / 70, epoch: 49 | loss: -4.7894578\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01319807 0.0015715  0.00050635]\n",
      "\t speed: 0.0268s/iter; left time: 96.5497s\n",
      "\t iters: 50 / 70, epoch: 49 | loss: -5.0872250\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02078418 0.02262292 0.01339709]\n",
      "\t speed: 0.0274s/iter; left time: 98.3524s\n",
      "\t iters: 60 / 70, epoch: 49 | loss: -5.0513563\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0005583  0.0029394  0.00105622]\n",
      "\t speed: 0.0288s/iter; left time: 103.0985s\n",
      "\t iters: 70 / 70, epoch: 49 | loss: -5.1560702\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0029076  0.00381709 0.00392648]\n",
      "\t speed: 0.0285s/iter; left time: 101.6245s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 49 Time: 2.013657569885254 Steps: 70\n",
      "Train Loss: -5.0887451 | Val Loss: -4.9126379\n",
      "\t iters: 10 / 70, epoch: 50 | loss: -5.0983381\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00029873 0.0023801  0.00134377]\n",
      "\t speed: 0.0272s/iter; left time: 96.7466s\n",
      "\t iters: 20 / 70, epoch: 50 | loss: -5.2068472\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00113553 0.00071261 0.00212013]\n",
      "\t speed: 0.0293s/iter; left time: 103.9405s\n",
      "\t iters: 30 / 70, epoch: 50 | loss: -5.1830015\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00299786 0.00169277 0.00059786]\n",
      "\t speed: 0.0378s/iter; left time: 133.7591s\n",
      "\t iters: 40 / 70, epoch: 50 | loss: -4.9137321\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01274305 0.02695938 0.02216704]\n",
      "\t speed: 0.0291s/iter; left time: 102.6701s\n",
      "\t iters: 50 / 70, epoch: 50 | loss: -4.8419800\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0016794  0.0010453  0.01207882]\n",
      "\t speed: 0.0291s/iter; left time: 102.2854s\n",
      "\t iters: 60 / 70, epoch: 50 | loss: -5.1251974\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0281031  0.01110764 0.03065108]\n",
      "\t speed: 0.0293s/iter; left time: 102.7529s\n",
      "\t iters: 70 / 70, epoch: 50 | loss: -5.1934996\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00954329 0.00615145 0.0142086 ]\n",
      "\t speed: 0.0273s/iter; left time: 95.4076s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 50 Time: 2.1392886638641357 Steps: 70\n",
      "Train Loss: -5.0967829 | Val Loss: -5.1209418\n",
      "\t iters: 10 / 70, epoch: 51 | loss: -5.2646132\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.006784   0.00367228 0.00195328]\n",
      "\t speed: 0.0267s/iter; left time: 93.2213s\n",
      "\t iters: 20 / 70, epoch: 51 | loss: -5.0853910\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00322938 0.00693559 0.00429007]\n",
      "\t speed: 0.0291s/iter; left time: 101.3334s\n",
      "\t iters: 30 / 70, epoch: 51 | loss: -5.0475073\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.000718   0.00081725 0.00870682]\n",
      "\t speed: 0.0269s/iter; left time: 93.2182s\n",
      "\t iters: 40 / 70, epoch: 51 | loss: -5.0668664\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00338993 0.00488549 0.00124685]\n",
      "\t speed: 0.0269s/iter; left time: 93.0831s\n",
      "\t iters: 50 / 70, epoch: 51 | loss: -4.8065171\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03392654 0.02978967 0.00451658]\n",
      "\t speed: 0.0278s/iter; left time: 95.9435s\n",
      "\t iters: 60 / 70, epoch: 51 | loss: -5.0470619\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00133336 0.0055969  0.00108888]\n",
      "\t speed: 0.0289s/iter; left time: 99.3322s\n",
      "\t iters: 70 / 70, epoch: 51 | loss: -4.9005928\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01223594 0.00417725 0.0143802 ]\n",
      "\t speed: 0.0274s/iter; left time: 94.0864s\n",
      "Validation loss did not decrease 7 / 20\n",
      "Epoch: 51 Time: 1.9869000911712646 Steps: 70\n",
      "Train Loss: -5.1045214 | Val Loss: -4.9965823\n",
      "\t iters: 10 / 70, epoch: 52 | loss: -5.2439828\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04833904 0.00230455 0.01046804]\n",
      "\t speed: 0.0344s/iter; left time: 117.6032s\n",
      "\t iters: 20 / 70, epoch: 52 | loss: -5.1461916\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00185498 0.00347751 0.00064942]\n",
      "\t speed: 0.0281s/iter; left time: 95.7857s\n",
      "\t iters: 30 / 70, epoch: 52 | loss: -4.8077703\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01222129 0.01767578 0.02098911]\n",
      "\t speed: 0.0287s/iter; left time: 97.7559s\n",
      "\t iters: 40 / 70, epoch: 52 | loss: -5.2641191\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01627615 0.00320051 0.00341819]\n",
      "\t speed: 0.0271s/iter; left time: 91.9519s\n",
      "\t iters: 50 / 70, epoch: 52 | loss: -4.9753194\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00984568 0.02259285 0.00948228]\n",
      "\t speed: 0.0280s/iter; left time: 94.6505s\n",
      "\t iters: 60 / 70, epoch: 52 | loss: -5.2642479\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00757531 0.00251682 0.00398092]\n",
      "\t speed: 0.0285s/iter; left time: 95.9653s\n",
      "\t iters: 70 / 70, epoch: 52 | loss: -4.9909902\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01105457 0.00225431 0.01376054]\n",
      "\t speed: 0.0289s/iter; left time: 97.2228s\n",
      "Validation loss did not decrease 8 / 20\n",
      "Epoch: 52 Time: 2.0853400230407715 Steps: 70\n",
      "Train Loss: -5.0898517 | Val Loss: -5.0009108\n",
      "\t iters: 10 / 70, epoch: 53 | loss: -4.8789320\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00110311 0.01110859 0.00107318]\n",
      "\t speed: 0.0253s/iter; left time: 84.8711s\n",
      "\t iters: 20 / 70, epoch: 53 | loss: -5.0532665\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00610132 0.00245031 0.00376638]\n",
      "\t speed: 0.0268s/iter; left time: 89.4195s\n",
      "\t iters: 30 / 70, epoch: 53 | loss: -5.2167482\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0240942  0.03163906 0.02817335]\n",
      "\t speed: 0.0268s/iter; left time: 89.1249s\n",
      "\t iters: 40 / 70, epoch: 53 | loss: -5.0580602\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00642877 0.00039654 0.00617755]\n",
      "\t speed: 0.0271s/iter; left time: 90.1553s\n",
      "\t iters: 50 / 70, epoch: 53 | loss: -4.8615770\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0010557  0.0072159  0.00139874]\n",
      "\t speed: 0.0271s/iter; left time: 89.6343s\n",
      "\t iters: 60 / 70, epoch: 53 | loss: -5.0894260\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01265102 0.02812146 0.02835537]\n",
      "\t speed: 0.0272s/iter; left time: 89.7602s\n",
      "\t iters: 70 / 70, epoch: 53 | loss: -5.3077507\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.001685   0.00082248 0.0004361 ]\n",
      "\t speed: 0.0359s/iter; left time: 118.1558s\n",
      "Validation loss did not decrease 9 / 20\n",
      "Epoch: 53 Time: 2.0109028816223145 Steps: 70\n",
      "Train Loss: -5.1089632 | Val Loss: -4.9593906\n",
      "\t iters: 10 / 70, epoch: 54 | loss: -5.3066478\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02071947 0.04216697 0.03233935]\n",
      "\t speed: 0.0258s/iter; left time: 84.8118s\n",
      "\t iters: 20 / 70, epoch: 54 | loss: -5.2791729\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00047268 0.0023622  0.00057697]\n",
      "\t speed: 0.0270s/iter; left time: 88.1836s\n",
      "\t iters: 30 / 70, epoch: 54 | loss: -5.2104750\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01574605 0.02561171 0.01426035]\n",
      "\t speed: 0.0272s/iter; left time: 88.8513s\n",
      "\t iters: 40 / 70, epoch: 54 | loss: -5.3917360\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00049397 0.00332947 0.00513468]\n",
      "\t speed: 0.0267s/iter; left time: 86.8738s\n",
      "\t iters: 50 / 70, epoch: 54 | loss: -4.9766936\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00743494 0.00749581 0.00506219]\n",
      "\t speed: 0.0271s/iter; left time: 87.9534s\n",
      "\t iters: 60 / 70, epoch: 54 | loss: -5.0170712\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00353426 0.00890829 0.02176508]\n",
      "\t speed: 0.0270s/iter; left time: 87.3824s\n",
      "\t iters: 70 / 70, epoch: 54 | loss: -4.9381466\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04505938 0.01029431 0.00536357]\n",
      "\t speed: 0.0282s/iter; left time: 90.9275s\n",
      "Validation loss did not decrease 10 / 20\n",
      "Epoch: 54 Time: 1.9421992301940918 Steps: 70\n",
      "Train Loss: -5.1133614 | Val Loss: -5.2041187\n",
      "\t iters: 10 / 70, epoch: 55 | loss: -5.1870074\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00473468 0.0054532  0.00355895]\n",
      "\t speed: 0.0264s/iter; left time: 84.8090s\n",
      "\t iters: 20 / 70, epoch: 55 | loss: -5.2873735\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00160735 0.00126873 0.00068478]\n",
      "\t speed: 0.0264s/iter; left time: 84.5077s\n",
      "\t iters: 30 / 70, epoch: 55 | loss: -4.8815112\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00370553 0.00069265 0.00380264]\n",
      "\t speed: 0.0270s/iter; left time: 86.3063s\n",
      "\t iters: 40 / 70, epoch: 55 | loss: -5.0880175\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00185344 0.00258114 0.0021754 ]\n",
      "\t speed: 0.0272s/iter; left time: 86.4208s\n",
      "\t iters: 50 / 70, epoch: 55 | loss: -5.0959005\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00566471 0.00868235 0.00286373]\n",
      "\t speed: 0.0363s/iter; left time: 114.9828s\n",
      "\t iters: 60 / 70, epoch: 55 | loss: -4.9831491\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00984283 0.03756351 0.01138096]\n",
      "\t speed: 0.0266s/iter; left time: 84.0656s\n",
      "\t iters: 70 / 70, epoch: 55 | loss: -5.0131922\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00047067 0.00448689 0.00067852]\n",
      "\t speed: 0.0269s/iter; left time: 84.7368s\n",
      "Validation loss did not decrease 11 / 20\n",
      "Epoch: 55 Time: 2.0179948806762695 Steps: 70\n",
      "Train Loss: -5.1049424 | Val Loss: -5.0951522\n",
      "\t iters: 10 / 70, epoch: 56 | loss: -5.1472721\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00925525 0.01733966 0.01594955]\n",
      "\t speed: 0.0263s/iter; left time: 82.7535s\n",
      "\t iters: 20 / 70, epoch: 56 | loss: -5.2441678\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00841263 0.01926231 0.01130419]\n",
      "\t speed: 0.0267s/iter; left time: 83.5740s\n",
      "\t iters: 30 / 70, epoch: 56 | loss: -4.9949384\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00047528 0.01032512 0.00174601]\n",
      "\t speed: 0.0268s/iter; left time: 83.6669s\n",
      "\t iters: 40 / 70, epoch: 56 | loss: -4.9555306\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.04558634 0.02191246 0.0256865 ]\n",
      "\t speed: 0.0281s/iter; left time: 87.3425s\n",
      "\t iters: 50 / 70, epoch: 56 | loss: -4.9599605\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00263664 0.0022577  0.01357762]\n",
      "\t speed: 0.0275s/iter; left time: 85.4048s\n",
      "\t iters: 60 / 70, epoch: 56 | loss: -5.1695900\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00984989 0.02947026 0.02912269]\n",
      "\t speed: 0.0287s/iter; left time: 88.8056s\n",
      "\t iters: 70 / 70, epoch: 56 | loss: -5.2334695\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03689133 0.01848736 0.0059997 ]\n",
      "\t speed: 0.0288s/iter; left time: 88.8469s\n",
      "Validation loss did not decrease 12 / 20\n",
      "Epoch: 56 Time: 1.9822125434875488 Steps: 70\n",
      "Train Loss: -5.1021759 | Val Loss: -5.0587718\n",
      "\t iters: 10 / 70, epoch: 57 | loss: -5.0706892\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01381855 0.02524344 0.00661706]\n",
      "\t speed: 0.0270s/iter; left time: 82.8201s\n",
      "\t iters: 20 / 70, epoch: 57 | loss: -5.2288094\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00165454 0.00492105 0.00114306]\n",
      "\t speed: 0.0283s/iter; left time: 86.6734s\n",
      "\t iters: 30 / 70, epoch: 57 | loss: -5.4137568\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0046456  0.00086144 0.00126736]\n",
      "\t speed: 0.0376s/iter; left time: 114.6157s\n",
      "\t iters: 40 / 70, epoch: 57 | loss: -5.3368769\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01901728 0.00124146 0.00300726]\n",
      "\t speed: 0.0281s/iter; left time: 85.3618s\n",
      "\t iters: 50 / 70, epoch: 57 | loss: -4.8012633\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00337988 0.00138036 0.00281389]\n",
      "\t speed: 0.0281s/iter; left time: 85.0447s\n",
      "\t iters: 60 / 70, epoch: 57 | loss: -5.2666469\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01462478 0.01414226 0.00962706]\n",
      "\t speed: 0.0284s/iter; left time: 85.7461s\n",
      "\t iters: 70 / 70, epoch: 57 | loss: -5.1637735\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01364586 0.02032938 0.0075898 ]\n",
      "\t speed: 0.0287s/iter; left time: 86.2659s\n",
      "Validation loss did not decrease 13 / 20\n",
      "Epoch: 57 Time: 2.1115589141845703 Steps: 70\n",
      "Train Loss: -5.1175743 | Val Loss: -5.0477935\n",
      "\t iters: 10 / 70, epoch: 58 | loss: -5.0546932\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00436877 0.00611872 0.00711481]\n",
      "\t speed: 0.0270s/iter; left time: 80.9253s\n",
      "\t iters: 20 / 70, epoch: 58 | loss: -5.1972284\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00510937 0.0105234  0.00652478]\n",
      "\t speed: 0.0292s/iter; left time: 87.2154s\n",
      "\t iters: 30 / 70, epoch: 58 | loss: -5.1876669\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00352676 0.00454815 0.00188474]\n",
      "\t speed: 0.0287s/iter; left time: 85.5844s\n",
      "\t iters: 40 / 70, epoch: 58 | loss: -5.1217198\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00613649 0.01684388 0.00067443]\n",
      "\t speed: 0.0291s/iter; left time: 86.3169s\n",
      "\t iters: 50 / 70, epoch: 58 | loss: -5.1387472\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00466114 0.01394172 0.01389825]\n",
      "\t speed: 0.0294s/iter; left time: 86.9487s\n",
      "\t iters: 60 / 70, epoch: 58 | loss: -5.2360463\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00227606 0.00313196 0.00404804]\n",
      "\t speed: 0.0291s/iter; left time: 86.0107s\n",
      "\t iters: 70 / 70, epoch: 58 | loss: -5.0490580\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00040581 0.00040054 0.00040703]\n",
      "\t speed: 0.0291s/iter; left time: 85.7061s\n",
      "Validation loss did not decrease 14 / 20\n",
      "Epoch: 58 Time: 2.066551685333252 Steps: 70\n",
      "Train Loss: -5.1363235 | Val Loss: -5.1683529\n",
      "\t iters: 10 / 70, epoch: 59 | loss: -5.2284374\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00240445 0.00031718 0.00309848]\n",
      "\t speed: 0.0345s/iter; left time: 101.0517s\n",
      "\t iters: 20 / 70, epoch: 59 | loss: -5.2202315\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03775802 0.01650829 0.00793976]\n",
      "\t speed: 0.0265s/iter; left time: 77.4856s\n",
      "\t iters: 30 / 70, epoch: 59 | loss: -5.0990968\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00493987 0.00063572 0.00896323]\n",
      "\t speed: 0.0268s/iter; left time: 77.9730s\n",
      "\t iters: 40 / 70, epoch: 59 | loss: -5.2623463\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00430498 0.00263075 0.00277845]\n",
      "\t speed: 0.0281s/iter; left time: 81.3986s\n",
      "\t iters: 50 / 70, epoch: 59 | loss: -5.1394262\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01410302 0.00943505 0.04835091]\n",
      "\t speed: 0.0286s/iter; left time: 82.5564s\n",
      "\t iters: 60 / 70, epoch: 59 | loss: -5.0681419\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00130738 0.00124477 0.00169936]\n",
      "\t speed: 0.0287s/iter; left time: 82.8043s\n",
      "\t iters: 70 / 70, epoch: 59 | loss: -5.1980081\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0062857  0.00448138 0.0046566 ]\n",
      "\t speed: 0.0286s/iter; left time: 82.2126s\n",
      "Validation loss did not decrease 15 / 20\n",
      "Epoch: 59 Time: 2.0663654804229736 Steps: 70\n",
      "Train Loss: -5.1310983 | Val Loss: -5.0686207\n",
      "\t iters: 10 / 70, epoch: 60 | loss: -4.9918365\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00317623 0.00022704 0.00670121]\n",
      "\t speed: 0.0249s/iter; left time: 71.1297s\n",
      "\t iters: 20 / 70, epoch: 60 | loss: -4.9969292\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00059713 0.00194348 0.00250903]\n",
      "\t speed: 0.0268s/iter; left time: 76.2947s\n",
      "\t iters: 30 / 70, epoch: 60 | loss: -5.2514777\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00655415 0.02648612 0.00707502]\n",
      "\t speed: 0.0271s/iter; left time: 77.0675s\n",
      "\t iters: 40 / 70, epoch: 60 | loss: -5.2228494\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00320982 0.01316793 0.03697317]\n",
      "\t speed: 0.0271s/iter; left time: 76.7110s\n",
      "\t iters: 50 / 70, epoch: 60 | loss: -5.0746899\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00720408 0.00227971 0.00142403]\n",
      "\t speed: 0.0271s/iter; left time: 76.3807s\n",
      "\t iters: 60 / 70, epoch: 60 | loss: -4.9657526\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00047094 0.00182043 0.00044762]\n",
      "\t speed: 0.0273s/iter; left time: 76.8093s\n",
      "\t iters: 70 / 70, epoch: 60 | loss: -4.8868418\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00707857 0.00820897 0.00435705]\n",
      "\t speed: 0.0364s/iter; left time: 101.8426s\n",
      "Validation loss did not decrease 16 / 20\n",
      "Epoch: 60 Time: 2.0162763595581055 Steps: 70\n",
      "Train Loss: -5.1493360 | Val Loss: -5.1473598\n",
      "\t iters: 10 / 70, epoch: 61 | loss: -5.0060420\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01319338 0.01989402 0.02732542]\n",
      "\t speed: 0.0257s/iter; left time: 71.7210s\n",
      "\t iters: 20 / 70, epoch: 61 | loss: -5.2350960\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02917214 0.03320127 0.03045818]\n",
      "\t speed: 0.0273s/iter; left time: 76.0151s\n",
      "\t iters: 30 / 70, epoch: 61 | loss: -5.0463343\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0213508  0.00129677 0.0110255 ]\n",
      "\t speed: 0.0267s/iter; left time: 74.1041s\n",
      "\t iters: 40 / 70, epoch: 61 | loss: -4.9421997\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00116173 0.01219697 0.00509129]\n",
      "\t speed: 0.0279s/iter; left time: 77.0174s\n",
      "\t iters: 50 / 70, epoch: 61 | loss: -5.1072807\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00495794 0.00985258 0.00816403]\n",
      "\t speed: 0.0288s/iter; left time: 79.2847s\n",
      "\t iters: 60 / 70, epoch: 61 | loss: -4.9076252\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02698717 0.02289234 0.01262059]\n",
      "\t speed: 0.0287s/iter; left time: 78.7062s\n",
      "\t iters: 70 / 70, epoch: 61 | loss: -5.1813393\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01544341 0.01890569 0.036104  ]\n",
      "\t speed: 0.0273s/iter; left time: 74.5909s\n",
      "Validation loss did not decrease 17 / 20\n",
      "Epoch: 61 Time: 1.9767675399780273 Steps: 70\n",
      "Train Loss: -5.1385359 | Val Loss: -5.2109995\n",
      "\t iters: 10 / 70, epoch: 62 | loss: -5.2475591\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01021986 0.00286348 0.00556248]\n",
      "\t speed: 0.0260s/iter; left time: 70.7367s\n",
      "\t iters: 20 / 70, epoch: 62 | loss: -4.9587827\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02697346 0.00347255 0.01356057]\n",
      "\t speed: 0.0270s/iter; left time: 73.3321s\n",
      "\t iters: 30 / 70, epoch: 62 | loss: -5.1346469\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00084068 0.00261803 0.00300493]\n",
      "\t speed: 0.0272s/iter; left time: 73.4706s\n",
      "\t iters: 40 / 70, epoch: 62 | loss: -4.8785601\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00496282 0.02661888 0.03008028]\n",
      "\t speed: 0.0272s/iter; left time: 73.1791s\n",
      "\t iters: 50 / 70, epoch: 62 | loss: -5.0224018\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00353677 0.00385545 0.00225448]\n",
      "\t speed: 0.0272s/iter; left time: 72.8725s\n",
      "\t iters: 60 / 70, epoch: 62 | loss: -5.1802168\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0088563  0.00970427 0.03229077]\n",
      "\t speed: 0.0372s/iter; left time: 99.3343s\n",
      "\t iters: 70 / 70, epoch: 62 | loss: -5.0741248\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00847947 0.00226174 0.00626991]\n",
      "\t speed: 0.0268s/iter; left time: 71.3480s\n",
      "Validation loss did not decrease 18 / 20\n",
      "Epoch: 62 Time: 2.036932945251465 Steps: 70\n",
      "Train Loss: -5.1604666 | Val Loss: -5.2804133\n",
      "\t iters: 10 / 70, epoch: 63 | loss: -5.2309260\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00273608 0.00557346 0.01804019]\n",
      "\t speed: 0.0260s/iter; left time: 69.0546s\n",
      "\t iters: 20 / 70, epoch: 63 | loss: -5.0729446\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00684828 0.00768226 0.01019651]\n",
      "\t speed: 0.0271s/iter; left time: 71.5511s\n",
      "\t iters: 30 / 70, epoch: 63 | loss: -5.1707697\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00312694 0.0146928  0.01361012]\n",
      "\t speed: 0.0268s/iter; left time: 70.5558s\n",
      "\t iters: 40 / 70, epoch: 63 | loss: -5.0741711\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02406938 0.01419915 0.03437889]\n",
      "\t speed: 0.0275s/iter; left time: 72.1269s\n",
      "\t iters: 50 / 70, epoch: 63 | loss: -4.8450737\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01502738 0.00870284 0.00473548]\n",
      "\t speed: 0.0278s/iter; left time: 72.6934s\n",
      "\t iters: 60 / 70, epoch: 63 | loss: -5.0350971\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02357785 0.03260697 0.03185066]\n",
      "\t speed: 0.0280s/iter; left time: 72.7460s\n",
      "\t iters: 70 / 70, epoch: 63 | loss: -5.2151175\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00207268 0.00320277 0.00033707]\n",
      "\t speed: 0.0279s/iter; left time: 72.2491s\n",
      "Validation loss did not decrease 19 / 20\n",
      "Epoch: 63 Time: 1.9624032974243164 Steps: 70\n",
      "Train Loss: -5.1518668 | Val Loss: -5.1998274\n",
      "\t iters: 10 / 70, epoch: 64 | loss: -5.1227641\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.02626702 0.00928536 0.01133419]\n",
      "\t speed: 0.0263s/iter; left time: 67.9156s\n",
      "\t iters: 20 / 70, epoch: 64 | loss: -5.0380011\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00042946 0.00075745 0.00268361]\n",
      "\t speed: 0.0277s/iter; left time: 71.3173s\n",
      "\t iters: 30 / 70, epoch: 64 | loss: -5.1769133\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.03388938 0.04061079 0.02259094]\n",
      "\t speed: 0.0279s/iter; left time: 71.4252s\n",
      "\t iters: 40 / 70, epoch: 64 | loss: -5.3191671\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01876117 0.00673568 0.00356697]\n",
      "\t speed: 0.0379s/iter; left time: 96.6153s\n",
      "\t iters: 50 / 70, epoch: 64 | loss: -5.2272906\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.01333878 0.00850248 0.00957283]\n",
      "\t speed: 0.0279s/iter; left time: 70.8216s\n",
      "\t iters: 60 / 70, epoch: 64 | loss: -5.0332241\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.00133984 0.00056536 0.00193437]\n",
      "\t speed: 0.0281s/iter; left time: 71.2148s\n",
      "\t iters: 70 / 70, epoch: 64 | loss: -5.2977505\n",
      "\t variance shape: torch.Size([256, 3])\n",
      "\t variance:  [0.0026538  0.00050839 0.0127065 ]\n",
      "\t speed: 0.0283s/iter; left time: 71.2674s\n",
      "Validation loss did not decrease 20 / 20\n",
      "Early stopping...\n"
     ]
    }
   ],
   "source": [
    "epochs = 100\n",
    "criterion =  ExpLikeliLoss(num_samples)\n",
    "model_optim = torch.optim.Adam(model.parameters(), lr=0.0002, betas=(0, 0.9))\n",
    "\n",
    "# define params for training\n",
    "TRAIN_STEPS = len(train_data_loader)\n",
    "early_stop = EarlyStop(20, 1e-6)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    iter_count = 0\n",
    "    train_loss = []\n",
    "    \n",
    "    epoch_time = time.time()\n",
    "    curr_time = time.time()\n",
    "    \n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(train_data_loader):\n",
    "        iter_count += 1\n",
    "        # zero-out grad\n",
    "        model_optim.zero_grad()\n",
    "        pred, true, logvar = process_batch(subj_id=subj_id,\n",
    "                                            batch_x=batch_x, \n",
    "                                            batch_y=batch_y, \n",
    "                                            batch_x_mark=batch_x_mark, \n",
    "                                            batch_y_mark=batch_y_mark, \n",
    "                                            len_pred=len_pred, \n",
    "                                            len_label=len_label, \n",
    "                                            model=model, \n",
    "                                            device=device)\n",
    "        loss = criterion(pred, true, logvar)\n",
    "        train_loss.append(float(loss.item()))\n",
    "        \n",
    "        # print every 10\n",
    "        if (i+1) % 10==0:\n",
    "            print(\"\\t iters: {0} / {3}, epoch: {1} | loss: {2:.7f}\".format(i + 1, epoch + 1, loss.item(), TRAIN_STEPS))\n",
    "            logvar = logvar.reshape(-1, num_samples)\n",
    "            print(f\"\\t variance shape: {logvar.shape}\")\n",
    "            print(\"\\t variance: \", np.exp(logvar.detach().cpu().numpy()[0, :]))\n",
    "            speed = (time.time() - curr_time) / iter_count\n",
    "            left_time = speed * ((epochs - epoch) * TRAIN_STEPS - i)\n",
    "            print('\\t speed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time))\n",
    "            iter_count = 0\n",
    "            curr_time = time.time()\n",
    "        \n",
    "        loss.backward()\n",
    "        model_optim.step()\n",
    "    # compute average train loss\n",
    "    train_loss = np.average(train_loss)\n",
    "\n",
    "    # compute validation loss\n",
    "    with torch.no_grad():\n",
    "        val_loss = []\n",
    "        for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(val_data_loader):\n",
    "            pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                    batch_x=batch_x, \n",
    "                                    batch_y=batch_y, \n",
    "                                    batch_x_mark=batch_x_mark, \n",
    "                                    batch_y_mark=batch_y_mark, \n",
    "                                    len_pred=len_pred, \n",
    "                                    len_label=len_label, \n",
    "                                    model=model, \n",
    "                                    device=device)\n",
    "            loss = criterion(pred, true, logvar)\n",
    "            val_loss.append(float(loss.item()))\n",
    "        val_loss = np.average(val_loss)\n",
    "    \n",
    "    # check early stopping\n",
    "    early_stop(val_loss, model, model_path)\n",
    "    if early_stop.stop:\n",
    "        print(\"Early stopping...\")\n",
    "        break\n",
    "\n",
    "    # update lr\n",
    "    # adjust_learning_rate(model_optim, epoch, lr)\n",
    "    \n",
    "    print(\"Epoch: {0} Time: {1} Steps: {2}\".format(epoch+1, time.time() - epoch_time, TRAIN_STEPS))\n",
    "    print(\"Train Loss: {0:.7f} | Val Loss: {1:.7f}\".format(train_loss, val_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test infinite mixture model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APE: 0.2863\n",
      "RMSE: 0.1355\n"
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "calibration = [[] for i in range(len_pred)]\n",
    "ape, rmse = [], []\n",
    "with torch.no_grad():\n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(test_data_loader):\n",
    "        pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                batch_x=batch_x, \n",
    "                                batch_y=batch_y, \n",
    "                                batch_x_mark=batch_x_mark, \n",
    "                                batch_y_mark=batch_y_mark, \n",
    "                                len_pred=len_pred, \n",
    "                                len_label=len_label, \n",
    "                                model=model, \n",
    "                                device=device)\n",
    "        \n",
    "        # arrange in proper shape: take mean of predicted samples\n",
    "        pred = pred.detach().cpu().numpy(); true = true.detach().cpu().numpy(); logvar = logvar.detach().cpu().numpy()\n",
    "        pred = pred.transpose((1,0,2)).reshape((pred.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        true = true.transpose((1,0,2)).reshape((true.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        logvar = logvar.transpose((1,0,2)).reshape((logvar.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        \n",
    "        # calculate calibration\n",
    "        for k in range(batch_size):\n",
    "            for i in range(len_pred):\n",
    "                ps = [norm.cdf(true[k, i, 0], pred[k, i, j], np.sqrt(np.exp(logvar[k, 0, j]))) \n",
    "                        for j in range(num_samples)]\n",
    "                p = np.average(ps)\n",
    "                calibration[i].append(p)\n",
    "\n",
    "        # for metrics: take mean of smaples, take one sample of true\n",
    "        pred = np.mean(pred, axis=2)\n",
    "        true = true[:, :, 0]\n",
    "        # compute APE / RMSE\n",
    "        ape.append(np.mean(np.abs(true - pred) / true))\n",
    "        rmse.append(np.sqrt(np.mean((true - pred)**2)))\n",
    " \n",
    "rmse = np.median(rmse)\n",
    "ape = np.median(ape)\n",
    "print(\"APE: {0:.4f}\".format(ape))\n",
    "print(\"RMSE: {0:.4f}\".format(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save for plotting\n",
    "num_samples=5 # increase number of samples from posterior\n",
    "collate_fn_custom = modify_collate(num_samples)\n",
    "test_data_loader = DataLoader(test_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=False, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True,\n",
    "                                collate_fn = collate_fn_custom)\n",
    "\n",
    "with torch.no_grad():\n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(test_data_loader):\n",
    "        pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                batch_x=batch_x, \n",
    "                                batch_y=batch_y, \n",
    "                                batch_x_mark=batch_x_mark, \n",
    "                                batch_y_mark=batch_y_mark, \n",
    "                                len_pred=len_pred, \n",
    "                                len_label=len_label, \n",
    "                                model=model, \n",
    "                                device=device)\n",
    "        \n",
    "        # arrange in proper shape: take mean of predicted samples\n",
    "        pred = pred.detach().cpu().numpy(); true = true.detach().cpu().numpy()\n",
    "        batch_x = batch_x.detach().cpu().numpy(); logvar = logvar.detach().cpu().numpy()\n",
    "        batch_x_mark = batch_x_mark.detach().cpu().numpy()\n",
    "        pred = pred.transpose((1,0,2)).reshape((pred.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        true = true.transpose((1,0,2)).reshape((true.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        logvar = logvar.transpose((1,0,2)).reshape((logvar.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        batch_x = batch_x.transpose((1,0,2)).reshape((batch_x.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        batch_x_mark = batch_x_mark.transpose((1,0,2)).reshape((batch_x_mark.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        break\n",
    "# save\n",
    "np.save(f\"{cache_path}/pred_mean_infmixt.npy\", pred)\n",
    "np.save(f\"{cache_path}/true_mean_infmixt.npy\", true)\n",
    "np.save(f\"{cache_path}/pred_logvar_infmixt.npy\", logvar)\n",
    "np.save(f\"{cache_path}/input_infmixt.npy\", batch_x)\n",
    "np.save(f\"{cache_path}/input_x_norm.npy\", batch_x_mark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0')\n",
    "model = Gluformer(d_model=128, \n",
    "                    n_heads=4, \n",
    "                    d_fcn=64, \n",
    "                    r_drop=0., \n",
    "                    activ=\"relu\", \n",
    "                    num_enc_layers=2, \n",
    "                    num_dec_layers=1,\n",
    "                    distil=True,\n",
    "                    len_seq=len_seq, \n",
    "                    len_pred=len_pred,\n",
    "                    num_features=1)\n",
    "model.train()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with the MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = os.path.join(save_path, \"model_norm.pth\")\n",
    "num_samples = 1 \n",
    "batch_size = 256 # batch size for optimization\n",
    "collate_fn_custom = modify_collate(num_samples)\n",
    "\n",
    "train_data_loader = DataLoader(train_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=True, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True, \n",
    "                                collate_fn = collate_fn_custom)\n",
    "\n",
    "val_data_loader = DataLoader(val_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=True, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True, \n",
    "                                collate_fn = collate_fn_custom)\n",
    "\n",
    "test_data_loader = DataLoader(test_data, \n",
    "                                batch_size=batch_size, \n",
    "                                shuffle=False, \n",
    "                                num_workers=0, \n",
    "                                drop_last=True,\n",
    "                                collate_fn = collate_fn_custom)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t iters: 10 / 70, epoch: 1 | loss: 0.0447731\n",
      "\t speed: 0.0199s/iter; left time: 139.1855s\n",
      "\t iters: 20 / 70, epoch: 1 | loss: 0.0183842\n",
      "\t speed: 0.0213s/iter; left time: 148.6249s\n",
      "\t iters: 30 / 70, epoch: 1 | loss: 0.0224471\n",
      "\t speed: 0.0217s/iter; left time: 151.3683s\n",
      "\t iters: 40 / 70, epoch: 1 | loss: 0.0465177\n",
      "\t speed: 0.0221s/iter; left time: 153.6261s\n",
      "\t iters: 50 / 70, epoch: 1 | loss: 0.0219864\n",
      "\t speed: 0.0241s/iter; left time: 167.1843s\n",
      "\t iters: 60 / 70, epoch: 1 | loss: 0.0204375\n",
      "\t speed: 0.0241s/iter; left time: 167.5306s\n",
      "\t iters: 70 / 70, epoch: 1 | loss: 0.0421068\n",
      "\t speed: 0.0243s/iter; left time: 168.1835s\n",
      "Validation loss descreased: inf -> 0.05124042679866155\n",
      "Epoch: 1 Time: 1.6331822872161865 Steps: 70\n",
      "Train Loss: 0.0525830 | Val Loss: 0.0512404\n",
      "\t iters: 10 / 70, epoch: 2 | loss: 0.0211044\n",
      "\t speed: 0.0204s/iter; left time: 141.4848s\n",
      "\t iters: 20 / 70, epoch: 2 | loss: 0.0268557\n",
      "\t speed: 0.0237s/iter; left time: 163.9251s\n",
      "\t iters: 30 / 70, epoch: 2 | loss: 0.0296156\n",
      "\t speed: 0.0240s/iter; left time: 165.4828s\n",
      "\t iters: 40 / 70, epoch: 2 | loss: 0.0181483\n",
      "\t speed: 0.0216s/iter; left time: 148.7374s\n",
      "\t iters: 50 / 70, epoch: 2 | loss: 0.0210600\n",
      "\t speed: 0.0220s/iter; left time: 151.3866s\n",
      "\t iters: 60 / 70, epoch: 2 | loss: 0.0315457\n",
      "\t speed: 0.0217s/iter; left time: 148.7613s\n",
      "\t iters: 70 / 70, epoch: 2 | loss: 0.0242411\n",
      "\t speed: 0.0220s/iter; left time: 151.0438s\n",
      "Validation loss descreased: 0.05124042679866155 -> 0.025496486574411392\n",
      "Epoch: 2 Time: 1.605712652206421 Steps: 70\n",
      "Train Loss: 0.0296397 | Val Loss: 0.0254965\n",
      "\t iters: 10 / 70, epoch: 3 | loss: 0.0367210\n",
      "\t speed: 0.0210s/iter; left time: 143.6319s\n",
      "\t iters: 20 / 70, epoch: 3 | loss: 0.0244855\n",
      "\t speed: 0.0219s/iter; left time: 150.0824s\n",
      "\t iters: 30 / 70, epoch: 3 | loss: 0.0458004\n",
      "\t speed: 0.0220s/iter; left time: 150.0574s\n",
      "\t iters: 40 / 70, epoch: 3 | loss: 0.0195005\n",
      "\t speed: 0.0221s/iter; left time: 150.6263s\n",
      "\t iters: 50 / 70, epoch: 3 | loss: 0.0442412\n",
      "\t speed: 0.0206s/iter; left time: 140.4406s\n",
      "\t iters: 60 / 70, epoch: 3 | loss: 0.0190720\n",
      "\t speed: 0.0207s/iter; left time: 140.8295s\n",
      "\t iters: 70 / 70, epoch: 3 | loss: 0.0483424\n",
      "\t speed: 0.0210s/iter; left time: 142.3045s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 3 Time: 1.5278685092926025 Steps: 70\n",
      "Train Loss: 0.0255731 | Val Loss: 0.0343830\n",
      "\t iters: 10 / 70, epoch: 4 | loss: 0.0206095\n",
      "\t speed: 0.0192s/iter; left time: 130.1462s\n",
      "\t iters: 20 / 70, epoch: 4 | loss: 0.0226144\n",
      "\t speed: 0.0211s/iter; left time: 142.8650s\n",
      "\t iters: 30 / 70, epoch: 4 | loss: 0.0182762\n",
      "\t speed: 0.0204s/iter; left time: 138.2057s\n",
      "\t iters: 40 / 70, epoch: 4 | loss: 0.0263386\n",
      "\t speed: 0.0219s/iter; left time: 147.9728s\n",
      "\t iters: 50 / 70, epoch: 4 | loss: 0.0232027\n",
      "\t speed: 0.0223s/iter; left time: 150.1492s\n",
      "\t iters: 60 / 70, epoch: 4 | loss: 0.0206441\n",
      "\t speed: 0.0220s/iter; left time: 148.1586s\n",
      "\t iters: 70 / 70, epoch: 4 | loss: 0.0199485\n",
      "\t speed: 0.0212s/iter; left time: 142.3369s\n",
      "Validation loss descreased: 0.025496486574411392 -> 0.020759391287962597\n",
      "Epoch: 4 Time: 1.5281453132629395 Steps: 70\n",
      "Train Loss: 0.0234080 | Val Loss: 0.0207594\n",
      "\t iters: 10 / 70, epoch: 5 | loss: 0.0247080\n",
      "\t speed: 0.0203s/iter; left time: 136.3740s\n",
      "\t iters: 20 / 70, epoch: 5 | loss: 0.0177881\n",
      "\t speed: 0.0209s/iter; left time: 140.2988s\n",
      "\t iters: 30 / 70, epoch: 5 | loss: 0.0244033\n",
      "\t speed: 0.0214s/iter; left time: 143.1442s\n",
      "\t iters: 40 / 70, epoch: 5 | loss: 0.0236817\n",
      "\t speed: 0.0216s/iter; left time: 144.2329s\n",
      "\t iters: 50 / 70, epoch: 5 | loss: 0.0204561\n",
      "\t speed: 0.0212s/iter; left time: 141.1794s\n",
      "\t iters: 60 / 70, epoch: 5 | loss: 0.0208672\n",
      "\t speed: 0.0214s/iter; left time: 142.6024s\n",
      "\t iters: 70 / 70, epoch: 5 | loss: 0.0212319\n",
      "\t speed: 0.0213s/iter; left time: 141.9389s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 5 Time: 1.5157785415649414 Steps: 70\n",
      "Train Loss: 0.0223186 | Val Loss: 0.0216800\n",
      "\t iters: 10 / 70, epoch: 6 | loss: 0.0195196\n",
      "\t speed: 0.0199s/iter; left time: 131.9028s\n",
      "\t iters: 20 / 70, epoch: 6 | loss: 0.0242195\n",
      "\t speed: 0.0213s/iter; left time: 141.1006s\n",
      "\t iters: 30 / 70, epoch: 6 | loss: 0.0213908\n",
      "\t speed: 0.0204s/iter; left time: 134.7435s\n",
      "\t iters: 40 / 70, epoch: 6 | loss: 0.0225077\n",
      "\t speed: 0.0214s/iter; left time: 141.5938s\n",
      "\t iters: 50 / 70, epoch: 6 | loss: 0.0215689\n",
      "\t speed: 0.0213s/iter; left time: 140.3484s\n",
      "\t iters: 60 / 70, epoch: 6 | loss: 0.0216624\n",
      "\t speed: 0.0207s/iter; left time: 136.1356s\n",
      "\t iters: 70 / 70, epoch: 6 | loss: 0.0276404\n",
      "\t speed: 0.0213s/iter; left time: 139.9808s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 6 Time: 1.4951777458190918 Steps: 70\n",
      "Train Loss: 0.0214220 | Val Loss: 0.0267429\n",
      "\t iters: 10 / 70, epoch: 7 | loss: 0.0228812\n",
      "\t speed: 0.0196s/iter; left time: 128.5524s\n",
      "\t iters: 20 / 70, epoch: 7 | loss: 0.0200841\n",
      "\t speed: 0.0229s/iter; left time: 150.2873s\n",
      "\t iters: 30 / 70, epoch: 7 | loss: 0.0202943\n",
      "\t speed: 0.0230s/iter; left time: 150.8604s\n",
      "\t iters: 40 / 70, epoch: 7 | loss: 0.0247579\n",
      "\t speed: 0.0227s/iter; left time: 148.1665s\n",
      "\t iters: 50 / 70, epoch: 7 | loss: 0.0183291\n",
      "\t speed: 0.0215s/iter; left time: 140.1458s\n",
      "\t iters: 60 / 70, epoch: 7 | loss: 0.0219088\n",
      "\t speed: 0.0209s/iter; left time: 136.1926s\n",
      "\t iters: 70 / 70, epoch: 7 | loss: 0.0208794\n",
      "\t speed: 0.0204s/iter; left time: 132.7912s\n",
      "Validation loss descreased: 0.020759391287962597 -> 0.019475818301240604\n",
      "Epoch: 7 Time: 1.558858871459961 Steps: 70\n",
      "Train Loss: 0.0206742 | Val Loss: 0.0194758\n",
      "\t iters: 10 / 70, epoch: 8 | loss: 0.0181406\n",
      "\t speed: 0.0211s/iter; left time: 137.2033s\n",
      "\t iters: 20 / 70, epoch: 8 | loss: 0.0185972\n",
      "\t speed: 0.0214s/iter; left time: 139.0416s\n",
      "\t iters: 30 / 70, epoch: 8 | loss: 0.0222917\n",
      "\t speed: 0.0223s/iter; left time: 144.2731s\n",
      "\t iters: 40 / 70, epoch: 8 | loss: 0.0234512\n",
      "\t speed: 0.0220s/iter; left time: 142.3383s\n",
      "\t iters: 50 / 70, epoch: 8 | loss: 0.0191793\n",
      "\t speed: 0.0217s/iter; left time: 140.2401s\n",
      "\t iters: 60 / 70, epoch: 8 | loss: 0.0243365\n",
      "\t speed: 0.0216s/iter; left time: 139.6485s\n",
      "\t iters: 70 / 70, epoch: 8 | loss: 0.0280261\n",
      "\t speed: 0.0217s/iter; left time: 139.7375s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 8 Time: 1.5557844638824463 Steps: 70\n",
      "Train Loss: 0.0206487 | Val Loss: 0.0203543\n",
      "\t iters: 10 / 70, epoch: 9 | loss: 0.0182717\n",
      "\t speed: 0.0202s/iter; left time: 129.8727s\n",
      "\t iters: 20 / 70, epoch: 9 | loss: 0.0191923\n",
      "\t speed: 0.0214s/iter; left time: 137.3955s\n",
      "\t iters: 30 / 70, epoch: 9 | loss: 0.0228295\n",
      "\t speed: 0.0210s/iter; left time: 134.8015s\n",
      "\t iters: 40 / 70, epoch: 9 | loss: 0.0149330\n",
      "\t speed: 0.0220s/iter; left time: 140.7327s\n",
      "\t iters: 50 / 70, epoch: 9 | loss: 0.0252274\n",
      "\t speed: 0.0219s/iter; left time: 139.7422s\n",
      "\t iters: 60 / 70, epoch: 9 | loss: 0.0152657\n",
      "\t speed: 0.0209s/iter; left time: 133.2798s\n",
      "\t iters: 70 / 70, epoch: 9 | loss: 0.0270660\n",
      "\t speed: 0.0210s/iter; left time: 133.9059s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 9 Time: 1.5213236808776855 Steps: 70\n",
      "Train Loss: 0.0198429 | Val Loss: 0.0242238\n",
      "\t iters: 10 / 70, epoch: 10 | loss: 0.0228573\n",
      "\t speed: 0.0208s/iter; left time: 132.0220s\n",
      "\t iters: 20 / 70, epoch: 10 | loss: 0.0185291\n",
      "\t speed: 0.0224s/iter; left time: 142.3110s\n",
      "\t iters: 30 / 70, epoch: 10 | loss: 0.0227420\n",
      "\t speed: 0.0222s/iter; left time: 140.9449s\n",
      "\t iters: 40 / 70, epoch: 10 | loss: 0.0185593\n",
      "\t speed: 0.0221s/iter; left time: 140.1818s\n",
      "\t iters: 50 / 70, epoch: 10 | loss: 0.0205074\n",
      "\t speed: 0.0223s/iter; left time: 140.9033s\n",
      "\t iters: 60 / 70, epoch: 10 | loss: 0.0174145\n",
      "\t speed: 0.0224s/iter; left time: 141.3954s\n",
      "\t iters: 70 / 70, epoch: 10 | loss: 0.0176703\n",
      "\t speed: 0.0226s/iter; left time: 142.3860s\n",
      "Validation loss descreased: 0.019475818301240604 -> 0.018102747698624928\n",
      "Epoch: 10 Time: 1.5974981784820557 Steps: 70\n",
      "Train Loss: 0.0198572 | Val Loss: 0.0181027\n",
      "\t iters: 10 / 70, epoch: 11 | loss: 0.0199623\n",
      "\t speed: 0.0205s/iter; left time: 129.0098s\n",
      "\t iters: 20 / 70, epoch: 11 | loss: 0.0215014\n",
      "\t speed: 0.0221s/iter; left time: 138.5600s\n",
      "\t iters: 30 / 70, epoch: 11 | loss: 0.0175875\n",
      "\t speed: 0.0215s/iter; left time: 134.6995s\n",
      "\t iters: 40 / 70, epoch: 11 | loss: 0.0206964\n",
      "\t speed: 0.0215s/iter; left time: 134.4625s\n",
      "\t iters: 50 / 70, epoch: 11 | loss: 0.0190074\n",
      "\t speed: 0.0227s/iter; left time: 142.0730s\n",
      "\t iters: 60 / 70, epoch: 11 | loss: 0.0225272\n",
      "\t speed: 0.0229s/iter; left time: 142.6709s\n",
      "\t iters: 70 / 70, epoch: 11 | loss: 0.0178555\n",
      "\t speed: 0.0225s/iter; left time: 139.8913s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 11 Time: 1.5719521045684814 Steps: 70\n",
      "Train Loss: 0.0196613 | Val Loss: 0.0189695\n",
      "\t iters: 10 / 70, epoch: 12 | loss: 0.0160942\n",
      "\t speed: 0.0202s/iter; left time: 125.5504s\n",
      "\t iters: 20 / 70, epoch: 12 | loss: 0.0180798\n",
      "\t speed: 0.0216s/iter; left time: 133.9238s\n",
      "\t iters: 30 / 70, epoch: 12 | loss: 0.0230835\n",
      "\t speed: 0.0217s/iter; left time: 134.7368s\n",
      "\t iters: 40 / 70, epoch: 12 | loss: 0.0146775\n",
      "\t speed: 0.0220s/iter; left time: 136.2893s\n",
      "\t iters: 50 / 70, epoch: 12 | loss: 0.0165900\n",
      "\t speed: 0.0215s/iter; left time: 133.1489s\n",
      "\t iters: 60 / 70, epoch: 12 | loss: 0.0185079\n",
      "\t speed: 0.0208s/iter; left time: 128.1118s\n",
      "\t iters: 70 / 70, epoch: 12 | loss: 0.0176598\n",
      "\t speed: 0.0208s/iter; left time: 128.0941s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 12 Time: 1.5206091403961182 Steps: 70\n",
      "Train Loss: 0.0195787 | Val Loss: 0.0193929\n",
      "\t iters: 10 / 70, epoch: 13 | loss: 0.0221876\n",
      "\t speed: 0.0198s/iter; left time: 121.7631s\n",
      "\t iters: 20 / 70, epoch: 13 | loss: 0.0177605\n",
      "\t speed: 0.0210s/iter; left time: 129.0941s\n",
      "\t iters: 30 / 70, epoch: 13 | loss: 0.0173936\n",
      "\t speed: 0.0211s/iter; left time: 129.6420s\n",
      "\t iters: 40 / 70, epoch: 13 | loss: 0.0182645\n",
      "\t speed: 0.0213s/iter; left time: 130.6583s\n",
      "\t iters: 50 / 70, epoch: 13 | loss: 0.0189070\n",
      "\t speed: 0.0218s/iter; left time: 133.3014s\n",
      "\t iters: 60 / 70, epoch: 13 | loss: 0.0195699\n",
      "\t speed: 0.0218s/iter; left time: 132.9370s\n",
      "\t iters: 70 / 70, epoch: 13 | loss: 0.0167043\n",
      "\t speed: 0.0209s/iter; left time: 127.4781s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 13 Time: 1.5128076076507568 Steps: 70\n",
      "Train Loss: 0.0194003 | Val Loss: 0.0197771\n",
      "\t iters: 10 / 70, epoch: 14 | loss: 0.0161560\n",
      "\t speed: 0.0200s/iter; left time: 121.3282s\n",
      "\t iters: 20 / 70, epoch: 14 | loss: 0.0172477\n",
      "\t speed: 0.0218s/iter; left time: 132.3763s\n",
      "\t iters: 30 / 70, epoch: 14 | loss: 0.0182846\n",
      "\t speed: 0.0227s/iter; left time: 137.3624s\n",
      "\t iters: 40 / 70, epoch: 14 | loss: 0.0233250\n",
      "\t speed: 0.0202s/iter; left time: 122.1727s\n",
      "\t iters: 50 / 70, epoch: 14 | loss: 0.0187521\n",
      "\t speed: 0.0212s/iter; left time: 128.0255s\n",
      "\t iters: 60 / 70, epoch: 14 | loss: 0.0208141\n",
      "\t speed: 0.0213s/iter; left time: 128.2385s\n",
      "\t iters: 70 / 70, epoch: 14 | loss: 0.0175092\n",
      "\t speed: 0.0205s/iter; left time: 123.1843s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 14 Time: 1.5108771324157715 Steps: 70\n",
      "Train Loss: 0.0192276 | Val Loss: 0.0196499\n",
      "\t iters: 10 / 70, epoch: 15 | loss: 0.0194320\n",
      "\t speed: 0.0190s/iter; left time: 114.0650s\n",
      "\t iters: 20 / 70, epoch: 15 | loss: 0.0207245\n",
      "\t speed: 0.0215s/iter; left time: 128.7809s\n",
      "\t iters: 30 / 70, epoch: 15 | loss: 0.0235102\n",
      "\t speed: 0.0214s/iter; left time: 128.3197s\n",
      "\t iters: 40 / 70, epoch: 15 | loss: 0.0206280\n",
      "\t speed: 0.0233s/iter; left time: 139.4800s\n",
      "\t iters: 50 / 70, epoch: 15 | loss: 0.0222610\n",
      "\t speed: 0.0235s/iter; left time: 140.1997s\n",
      "\t iters: 60 / 70, epoch: 15 | loss: 0.0192450\n",
      "\t speed: 0.0212s/iter; left time: 126.4919s\n",
      "\t iters: 70 / 70, epoch: 15 | loss: 0.0177392\n",
      "\t speed: 0.0214s/iter; left time: 127.1654s\n",
      "Validation loss descreased: 0.018102747698624928 -> 0.01788031340887149\n",
      "Epoch: 15 Time: 1.5668787956237793 Steps: 70\n",
      "Train Loss: 0.0188321 | Val Loss: 0.0178803\n",
      "\t iters: 10 / 70, epoch: 16 | loss: 0.0154532\n",
      "\t speed: 0.0227s/iter; left time: 135.1184s\n",
      "\t iters: 20 / 70, epoch: 16 | loss: 0.0160491\n",
      "\t speed: 0.0240s/iter; left time: 142.1129s\n",
      "\t iters: 30 / 70, epoch: 16 | loss: 0.0165289\n",
      "\t speed: 0.0241s/iter; left time: 142.4734s\n",
      "\t iters: 40 / 70, epoch: 16 | loss: 0.0165161\n",
      "\t speed: 0.0240s/iter; left time: 142.0083s\n",
      "\t iters: 50 / 70, epoch: 16 | loss: 0.0202635\n",
      "\t speed: 0.0217s/iter; left time: 127.9010s\n",
      "\t iters: 60 / 70, epoch: 16 | loss: 0.0161706\n",
      "\t speed: 0.0211s/iter; left time: 124.5688s\n",
      "\t iters: 70 / 70, epoch: 16 | loss: 0.0204626\n",
      "\t speed: 0.0215s/iter; left time: 126.2454s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 16 Time: 1.6281065940856934 Steps: 70\n",
      "Train Loss: 0.0189114 | Val Loss: 0.0190292\n",
      "\t iters: 10 / 70, epoch: 17 | loss: 0.0223788\n",
      "\t speed: 0.0211s/iter; left time: 123.9888s\n",
      "\t iters: 20 / 70, epoch: 17 | loss: 0.0199670\n",
      "\t speed: 0.0211s/iter; left time: 123.8691s\n",
      "\t iters: 30 / 70, epoch: 17 | loss: 0.0210164\n",
      "\t speed: 0.0212s/iter; left time: 123.7803s\n",
      "\t iters: 40 / 70, epoch: 17 | loss: 0.0153855\n",
      "\t speed: 0.0215s/iter; left time: 125.3753s\n",
      "\t iters: 50 / 70, epoch: 17 | loss: 0.0177721\n",
      "\t speed: 0.0216s/iter; left time: 125.9916s\n",
      "\t iters: 60 / 70, epoch: 17 | loss: 0.0211444\n",
      "\t speed: 0.0242s/iter; left time: 140.7353s\n",
      "\t iters: 70 / 70, epoch: 17 | loss: 0.0229062\n",
      "\t speed: 0.0241s/iter; left time: 140.1149s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 17 Time: 1.5920569896697998 Steps: 70\n",
      "Train Loss: 0.0188299 | Val Loss: 0.0228756\n",
      "\t iters: 10 / 70, epoch: 18 | loss: 0.0179103\n",
      "\t speed: 0.0204s/iter; left time: 118.4260s\n",
      "\t iters: 20 / 70, epoch: 18 | loss: 0.0211592\n",
      "\t speed: 0.0215s/iter; left time: 124.2252s\n",
      "\t iters: 30 / 70, epoch: 18 | loss: 0.0169031\n",
      "\t speed: 0.0219s/iter; left time: 126.7855s\n",
      "\t iters: 40 / 70, epoch: 18 | loss: 0.0153223\n",
      "\t speed: 0.0241s/iter; left time: 139.3469s\n",
      "\t iters: 50 / 70, epoch: 18 | loss: 0.0169054\n",
      "\t speed: 0.0240s/iter; left time: 138.4159s\n",
      "\t iters: 60 / 70, epoch: 18 | loss: 0.0192325\n",
      "\t speed: 0.0225s/iter; left time: 129.6538s\n",
      "\t iters: 70 / 70, epoch: 18 | loss: 0.0173043\n",
      "\t speed: 0.0242s/iter; left time: 139.1031s\n",
      "Validation loss descreased: 0.01788031340887149 -> 0.01774362288415432\n",
      "Epoch: 18 Time: 1.6467840671539307 Steps: 70\n",
      "Train Loss: 0.0187872 | Val Loss: 0.0177436\n",
      "\t iters: 10 / 70, epoch: 19 | loss: 0.0219737\n",
      "\t speed: 0.0204s/iter; left time: 116.6686s\n",
      "\t iters: 20 / 70, epoch: 19 | loss: 0.0164884\n",
      "\t speed: 0.0221s/iter; left time: 126.2441s\n",
      "\t iters: 30 / 70, epoch: 19 | loss: 0.0238866\n",
      "\t speed: 0.0218s/iter; left time: 124.2777s\n",
      "\t iters: 40 / 70, epoch: 19 | loss: 0.0203762\n",
      "\t speed: 0.0214s/iter; left time: 122.0623s\n",
      "\t iters: 50 / 70, epoch: 19 | loss: 0.0205239\n",
      "\t speed: 0.0212s/iter; left time: 120.7702s\n",
      "\t iters: 60 / 70, epoch: 19 | loss: 0.0186896\n",
      "\t speed: 0.0216s/iter; left time: 122.7967s\n",
      "\t iters: 70 / 70, epoch: 19 | loss: 0.0205743\n",
      "\t speed: 0.0212s/iter; left time: 120.4477s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 19 Time: 1.5328993797302246 Steps: 70\n",
      "Train Loss: 0.0187941 | Val Loss: 0.0177604\n",
      "\t iters: 10 / 70, epoch: 20 | loss: 0.0194241\n",
      "\t speed: 0.0219s/iter; left time: 123.8603s\n",
      "\t iters: 20 / 70, epoch: 20 | loss: 0.0156370\n",
      "\t speed: 0.0212s/iter; left time: 119.8114s\n",
      "\t iters: 30 / 70, epoch: 20 | loss: 0.0199393\n",
      "\t speed: 0.0213s/iter; left time: 120.1168s\n",
      "\t iters: 40 / 70, epoch: 20 | loss: 0.0178506\n",
      "\t speed: 0.0216s/iter; left time: 121.8465s\n",
      "\t iters: 50 / 70, epoch: 20 | loss: 0.0204936\n",
      "\t speed: 0.0211s/iter; left time: 118.8690s\n",
      "\t iters: 60 / 70, epoch: 20 | loss: 0.0158559\n",
      "\t speed: 0.0213s/iter; left time: 119.4258s\n",
      "\t iters: 70 / 70, epoch: 20 | loss: 0.0147497\n",
      "\t speed: 0.0212s/iter; left time: 118.4981s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 20 Time: 1.5320959091186523 Steps: 70\n",
      "Train Loss: 0.0185299 | Val Loss: 0.0187197\n",
      "\t iters: 10 / 70, epoch: 21 | loss: 0.0195513\n",
      "\t speed: 0.0208s/iter; left time: 116.2220s\n",
      "\t iters: 20 / 70, epoch: 21 | loss: 0.0232020\n",
      "\t speed: 0.0219s/iter; left time: 122.0281s\n",
      "\t iters: 30 / 70, epoch: 21 | loss: 0.0194761\n",
      "\t speed: 0.0217s/iter; left time: 120.6271s\n",
      "\t iters: 40 / 70, epoch: 21 | loss: 0.0229413\n",
      "\t speed: 0.0218s/iter; left time: 120.9529s\n",
      "\t iters: 50 / 70, epoch: 21 | loss: 0.0183313\n",
      "\t speed: 0.0216s/iter; left time: 120.0990s\n",
      "\t iters: 60 / 70, epoch: 21 | loss: 0.0214158\n",
      "\t speed: 0.0219s/iter; left time: 121.1787s\n",
      "\t iters: 70 / 70, epoch: 21 | loss: 0.0198062\n",
      "\t speed: 0.0219s/iter; left time: 121.3763s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 21 Time: 1.5507476329803467 Steps: 70\n",
      "Train Loss: 0.0188171 | Val Loss: 0.0177570\n",
      "\t iters: 10 / 70, epoch: 22 | loss: 0.0148960\n",
      "\t speed: 0.0204s/iter; left time: 112.4815s\n",
      "\t iters: 20 / 70, epoch: 22 | loss: 0.0244748\n",
      "\t speed: 0.0220s/iter; left time: 121.0456s\n",
      "\t iters: 30 / 70, epoch: 22 | loss: 0.0170493\n",
      "\t speed: 0.0218s/iter; left time: 120.0903s\n",
      "\t iters: 40 / 70, epoch: 22 | loss: 0.0158494\n",
      "\t speed: 0.0206s/iter; left time: 113.2500s\n",
      "\t iters: 50 / 70, epoch: 22 | loss: 0.0181901\n",
      "\t speed: 0.0221s/iter; left time: 121.3639s\n",
      "\t iters: 60 / 70, epoch: 22 | loss: 0.0223636\n",
      "\t speed: 0.0222s/iter; left time: 121.5420s\n",
      "\t iters: 70 / 70, epoch: 22 | loss: 0.0177002\n",
      "\t speed: 0.0219s/iter; left time: 119.4310s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 22 Time: 1.545577049255371 Steps: 70\n",
      "Train Loss: 0.0184719 | Val Loss: 0.0192670\n",
      "\t iters: 10 / 70, epoch: 23 | loss: 0.0190301\n",
      "\t speed: 0.0207s/iter; left time: 112.7511s\n",
      "\t iters: 20 / 70, epoch: 23 | loss: 0.0196821\n",
      "\t speed: 0.0217s/iter; left time: 118.0946s\n",
      "\t iters: 30 / 70, epoch: 23 | loss: 0.0209565\n",
      "\t speed: 0.0193s/iter; left time: 105.0422s\n",
      "\t iters: 40 / 70, epoch: 23 | loss: 0.0172651\n",
      "\t speed: 0.0213s/iter; left time: 115.7151s\n",
      "\t iters: 50 / 70, epoch: 23 | loss: 0.0190482\n",
      "\t speed: 0.0214s/iter; left time: 115.9765s\n",
      "\t iters: 60 / 70, epoch: 23 | loss: 0.0197447\n",
      "\t speed: 0.0213s/iter; left time: 115.0397s\n",
      "\t iters: 70 / 70, epoch: 23 | loss: 0.0210918\n",
      "\t speed: 0.0220s/iter; left time: 118.7947s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 23 Time: 1.515397071838379 Steps: 70\n",
      "Train Loss: 0.0185524 | Val Loss: 0.0181461\n",
      "\t iters: 10 / 70, epoch: 24 | loss: 0.0164411\n",
      "\t speed: 0.0208s/iter; left time: 111.6665s\n",
      "\t iters: 20 / 70, epoch: 24 | loss: 0.0199032\n",
      "\t speed: 0.0220s/iter; left time: 118.1554s\n",
      "\t iters: 30 / 70, epoch: 24 | loss: 0.0198884\n",
      "\t speed: 0.0219s/iter; left time: 117.2390s\n",
      "\t iters: 40 / 70, epoch: 24 | loss: 0.0172551\n",
      "\t speed: 0.0220s/iter; left time: 117.4804s\n",
      "\t iters: 50 / 70, epoch: 24 | loss: 0.0189715\n",
      "\t speed: 0.0222s/iter; left time: 118.3274s\n",
      "\t iters: 60 / 70, epoch: 24 | loss: 0.0176465\n",
      "\t speed: 0.0222s/iter; left time: 118.4658s\n",
      "\t iters: 70 / 70, epoch: 24 | loss: 0.0170589\n",
      "\t speed: 0.0218s/iter; left time: 115.8906s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 24 Time: 1.5643565654754639 Steps: 70\n",
      "Train Loss: 0.0185760 | Val Loss: 0.0194346\n",
      "\t iters: 10 / 70, epoch: 25 | loss: 0.0186143\n",
      "\t speed: 0.0207s/iter; left time: 110.1883s\n",
      "\t iters: 20 / 70, epoch: 25 | loss: 0.0208856\n",
      "\t speed: 0.0220s/iter; left time: 116.8400s\n",
      "\t iters: 30 / 70, epoch: 25 | loss: 0.0162744\n",
      "\t speed: 0.0220s/iter; left time: 116.2239s\n",
      "\t iters: 40 / 70, epoch: 25 | loss: 0.0167761\n",
      "\t speed: 0.0206s/iter; left time: 109.0270s\n",
      "\t iters: 50 / 70, epoch: 25 | loss: 0.0181776\n",
      "\t speed: 0.0208s/iter; left time: 109.5851s\n",
      "\t iters: 60 / 70, epoch: 25 | loss: 0.0168736\n",
      "\t speed: 0.0209s/iter; left time: 109.6958s\n",
      "\t iters: 70 / 70, epoch: 25 | loss: 0.0163854\n",
      "\t speed: 0.0210s/iter; left time: 110.0841s\n",
      "Validation loss did not decrease 7 / 20\n",
      "Epoch: 25 Time: 1.5144507884979248 Steps: 70\n",
      "Train Loss: 0.0185109 | Val Loss: 0.0189117\n",
      "\t iters: 10 / 70, epoch: 26 | loss: 0.0220801\n",
      "\t speed: 0.0206s/iter; left time: 107.8598s\n",
      "\t iters: 20 / 70, epoch: 26 | loss: 0.0203060\n",
      "\t speed: 0.0220s/iter; left time: 115.2572s\n",
      "\t iters: 30 / 70, epoch: 26 | loss: 0.0208266\n",
      "\t speed: 0.0219s/iter; left time: 114.3967s\n",
      "\t iters: 40 / 70, epoch: 26 | loss: 0.0142026\n",
      "\t speed: 0.0219s/iter; left time: 114.2707s\n",
      "\t iters: 50 / 70, epoch: 26 | loss: 0.0198228\n",
      "\t speed: 0.0216s/iter; left time: 112.2981s\n",
      "\t iters: 60 / 70, epoch: 26 | loss: 0.0190651\n",
      "\t speed: 0.0209s/iter; left time: 108.2376s\n",
      "\t iters: 70 / 70, epoch: 26 | loss: 0.0190010\n",
      "\t speed: 0.0208s/iter; left time: 107.5863s\n",
      "Validation loss descreased: 0.01774362288415432 -> 0.017446365828315418\n",
      "Epoch: 26 Time: 1.544217586517334 Steps: 70\n",
      "Train Loss: 0.0183980 | Val Loss: 0.0174464\n",
      "\t iters: 10 / 70, epoch: 27 | loss: 0.0166085\n",
      "\t speed: 0.0202s/iter; left time: 104.3706s\n",
      "\t iters: 20 / 70, epoch: 27 | loss: 0.0187149\n",
      "\t speed: 0.0214s/iter; left time: 110.5784s\n",
      "\t iters: 30 / 70, epoch: 27 | loss: 0.0228854\n",
      "\t speed: 0.0217s/iter; left time: 111.8509s\n",
      "\t iters: 40 / 70, epoch: 27 | loss: 0.0188028\n",
      "\t speed: 0.0236s/iter; left time: 121.4110s\n",
      "\t iters: 50 / 70, epoch: 27 | loss: 0.0147552\n",
      "\t speed: 0.0230s/iter; left time: 117.7982s\n",
      "\t iters: 60 / 70, epoch: 27 | loss: 0.0188468\n",
      "\t speed: 0.0233s/iter; left time: 119.3492s\n",
      "\t iters: 70 / 70, epoch: 27 | loss: 0.0170830\n",
      "\t speed: 0.0230s/iter; left time: 117.4253s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 27 Time: 1.6006851196289062 Steps: 70\n",
      "Train Loss: 0.0184000 | Val Loss: 0.0194297\n",
      "\t iters: 10 / 70, epoch: 28 | loss: 0.0134082\n",
      "\t speed: 0.0203s/iter; left time: 103.4460s\n",
      "\t iters: 20 / 70, epoch: 28 | loss: 0.0172370\n",
      "\t speed: 0.0204s/iter; left time: 103.7837s\n",
      "\t iters: 30 / 70, epoch: 28 | loss: 0.0204007\n",
      "\t speed: 0.0214s/iter; left time: 108.6091s\n",
      "\t iters: 40 / 70, epoch: 28 | loss: 0.0179087\n",
      "\t speed: 0.0206s/iter; left time: 104.4209s\n",
      "\t iters: 50 / 70, epoch: 28 | loss: 0.0181019\n",
      "\t speed: 0.0210s/iter; left time: 106.2885s\n",
      "\t iters: 60 / 70, epoch: 28 | loss: 0.0165109\n",
      "\t speed: 0.0212s/iter; left time: 107.0065s\n",
      "\t iters: 70 / 70, epoch: 28 | loss: 0.0168529\n",
      "\t speed: 0.0210s/iter; left time: 106.0096s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 28 Time: 1.4953405857086182 Steps: 70\n",
      "Train Loss: 0.0184109 | Val Loss: 0.0187671\n",
      "\t iters: 10 / 70, epoch: 29 | loss: 0.0180923\n",
      "\t speed: 0.0201s/iter; left time: 101.0311s\n",
      "\t iters: 20 / 70, epoch: 29 | loss: 0.0192755\n",
      "\t speed: 0.0220s/iter; left time: 110.6016s\n",
      "\t iters: 30 / 70, epoch: 29 | loss: 0.0193963\n",
      "\t speed: 0.0220s/iter; left time: 110.3000s\n",
      "\t iters: 40 / 70, epoch: 29 | loss: 0.0186034\n",
      "\t speed: 0.0218s/iter; left time: 109.1806s\n",
      "\t iters: 50 / 70, epoch: 29 | loss: 0.0188591\n",
      "\t speed: 0.0218s/iter; left time: 108.6077s\n",
      "\t iters: 60 / 70, epoch: 29 | loss: 0.0205185\n",
      "\t speed: 0.0218s/iter; left time: 108.4674s\n",
      "\t iters: 70 / 70, epoch: 29 | loss: 0.0203765\n",
      "\t speed: 0.0219s/iter; left time: 108.9555s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 29 Time: 1.5496759414672852 Steps: 70\n",
      "Train Loss: 0.0183335 | Val Loss: 0.0192742\n",
      "\t iters: 10 / 70, epoch: 30 | loss: 0.0173922\n",
      "\t speed: 0.0205s/iter; left time: 101.9365s\n",
      "\t iters: 20 / 70, epoch: 30 | loss: 0.0172310\n",
      "\t speed: 0.0217s/iter; left time: 107.2463s\n",
      "\t iters: 30 / 70, epoch: 30 | loss: 0.0146511\n",
      "\t speed: 0.0216s/iter; left time: 106.9277s\n",
      "\t iters: 40 / 70, epoch: 30 | loss: 0.0197014\n",
      "\t speed: 0.0221s/iter; left time: 108.9103s\n",
      "\t iters: 50 / 70, epoch: 30 | loss: 0.0179793\n",
      "\t speed: 0.0218s/iter; left time: 107.3688s\n",
      "\t iters: 60 / 70, epoch: 30 | loss: 0.0168537\n",
      "\t speed: 0.0221s/iter; left time: 108.3646s\n",
      "\t iters: 70 / 70, epoch: 30 | loss: 0.0171843\n",
      "\t speed: 0.0220s/iter; left time: 107.9190s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 30 Time: 1.5543863773345947 Steps: 70\n",
      "Train Loss: 0.0183126 | Val Loss: 0.0190437\n",
      "\t iters: 10 / 70, epoch: 31 | loss: 0.0203917\n",
      "\t speed: 0.0207s/iter; left time: 101.0929s\n",
      "\t iters: 20 / 70, epoch: 31 | loss: 0.0181912\n",
      "\t speed: 0.0221s/iter; left time: 107.9229s\n",
      "\t iters: 30 / 70, epoch: 31 | loss: 0.0182243\n",
      "\t speed: 0.0219s/iter; left time: 106.4358s\n",
      "\t iters: 40 / 70, epoch: 31 | loss: 0.0190775\n",
      "\t speed: 0.0217s/iter; left time: 105.6513s\n",
      "\t iters: 50 / 70, epoch: 31 | loss: 0.0191038\n",
      "\t speed: 0.0219s/iter; left time: 106.0725s\n",
      "\t iters: 60 / 70, epoch: 31 | loss: 0.0217099\n",
      "\t speed: 0.0218s/iter; left time: 105.4020s\n",
      "\t iters: 70 / 70, epoch: 31 | loss: 0.0174297\n",
      "\t speed: 0.0218s/iter; left time: 105.2249s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 31 Time: 1.553541660308838 Steps: 70\n",
      "Train Loss: 0.0183619 | Val Loss: 0.0182863\n",
      "\t iters: 10 / 70, epoch: 32 | loss: 0.0189899\n",
      "\t speed: 0.0204s/iter; left time: 98.4977s\n",
      "\t iters: 20 / 70, epoch: 32 | loss: 0.0179494\n",
      "\t speed: 0.0219s/iter; left time: 105.2024s\n",
      "\t iters: 30 / 70, epoch: 32 | loss: 0.0181935\n",
      "\t speed: 0.0220s/iter; left time: 105.5134s\n",
      "\t iters: 40 / 70, epoch: 32 | loss: 0.0213901\n",
      "\t speed: 0.0218s/iter; left time: 104.6493s\n",
      "\t iters: 50 / 70, epoch: 32 | loss: 0.0177193\n",
      "\t speed: 0.0219s/iter; left time: 104.7898s\n",
      "\t iters: 60 / 70, epoch: 32 | loss: 0.0171468\n",
      "\t speed: 0.0221s/iter; left time: 105.4226s\n",
      "\t iters: 70 / 70, epoch: 32 | loss: 0.0152928\n",
      "\t speed: 0.0225s/iter; left time: 106.9686s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 32 Time: 1.5626256465911865 Steps: 70\n",
      "Train Loss: 0.0182293 | Val Loss: 0.0178287\n",
      "\t iters: 10 / 70, epoch: 33 | loss: 0.0184778\n",
      "\t speed: 0.0207s/iter; left time: 98.2494s\n",
      "\t iters: 20 / 70, epoch: 33 | loss: 0.0168273\n",
      "\t speed: 0.0219s/iter; left time: 103.8643s\n",
      "\t iters: 30 / 70, epoch: 33 | loss: 0.0169335\n",
      "\t speed: 0.0220s/iter; left time: 103.9145s\n",
      "\t iters: 40 / 70, epoch: 33 | loss: 0.0139980\n",
      "\t speed: 0.0220s/iter; left time: 103.8743s\n",
      "\t iters: 50 / 70, epoch: 33 | loss: 0.0155851\n",
      "\t speed: 0.0217s/iter; left time: 102.2489s\n",
      "\t iters: 60 / 70, epoch: 33 | loss: 0.0184147\n",
      "\t speed: 0.0218s/iter; left time: 102.5273s\n",
      "\t iters: 70 / 70, epoch: 33 | loss: 0.0148877\n",
      "\t speed: 0.0222s/iter; left time: 103.9431s\n",
      "Validation loss did not decrease 7 / 20\n",
      "Epoch: 33 Time: 1.5579309463500977 Steps: 70\n",
      "Train Loss: 0.0182752 | Val Loss: 0.0177594\n",
      "\t iters: 10 / 70, epoch: 34 | loss: 0.0173225\n",
      "\t speed: 0.0206s/iter; left time: 96.3274s\n",
      "\t iters: 20 / 70, epoch: 34 | loss: 0.0174711\n",
      "\t speed: 0.0225s/iter; left time: 105.0423s\n",
      "\t iters: 30 / 70, epoch: 34 | loss: 0.0173200\n",
      "\t speed: 0.0218s/iter; left time: 101.4470s\n",
      "\t iters: 40 / 70, epoch: 34 | loss: 0.0153960\n",
      "\t speed: 0.0218s/iter; left time: 101.5286s\n",
      "\t iters: 50 / 70, epoch: 34 | loss: 0.0160790\n",
      "\t speed: 0.0220s/iter; left time: 101.9522s\n",
      "\t iters: 60 / 70, epoch: 34 | loss: 0.0173896\n",
      "\t speed: 0.0222s/iter; left time: 102.5956s\n",
      "\t iters: 70 / 70, epoch: 34 | loss: 0.0159273\n",
      "\t speed: 0.0221s/iter; left time: 101.9719s\n",
      "Validation loss did not decrease 8 / 20\n",
      "Epoch: 34 Time: 1.564960241317749 Steps: 70\n",
      "Train Loss: 0.0182726 | Val Loss: 0.0179968\n",
      "\t iters: 10 / 70, epoch: 35 | loss: 0.0178432\n",
      "\t speed: 0.0209s/iter; left time: 96.1787s\n",
      "\t iters: 20 / 70, epoch: 35 | loss: 0.0180434\n",
      "\t speed: 0.0218s/iter; left time: 100.4177s\n",
      "\t iters: 30 / 70, epoch: 35 | loss: 0.0174449\n",
      "\t speed: 0.0218s/iter; left time: 100.0182s\n",
      "\t iters: 40 / 70, epoch: 35 | loss: 0.0170037\n",
      "\t speed: 0.0224s/iter; left time: 102.5988s\n",
      "\t iters: 50 / 70, epoch: 35 | loss: 0.0143454\n",
      "\t speed: 0.0194s/iter; left time: 88.6801s\n",
      "\t iters: 60 / 70, epoch: 35 | loss: 0.0140285\n",
      "\t speed: 0.0200s/iter; left time: 91.3141s\n",
      "\t iters: 70 / 70, epoch: 35 | loss: 0.0169394\n",
      "\t speed: 0.0212s/iter; left time: 96.6918s\n",
      "Validation loss did not decrease 9 / 20\n",
      "Epoch: 35 Time: 1.5163674354553223 Steps: 70\n",
      "Train Loss: 0.0181745 | Val Loss: 0.0179298\n",
      "\t iters: 10 / 70, epoch: 36 | loss: 0.0176027\n",
      "\t speed: 0.0232s/iter; left time: 105.4987s\n",
      "\t iters: 20 / 70, epoch: 36 | loss: 0.0183097\n",
      "\t speed: 0.0213s/iter; left time: 96.4844s\n",
      "\t iters: 30 / 70, epoch: 36 | loss: 0.0138420\n",
      "\t speed: 0.0214s/iter; left time: 96.5950s\n",
      "\t iters: 40 / 70, epoch: 36 | loss: 0.0197470\n",
      "\t speed: 0.0212s/iter; left time: 95.7863s\n",
      "\t iters: 50 / 70, epoch: 36 | loss: 0.0150454\n",
      "\t speed: 0.0220s/iter; left time: 99.0192s\n",
      "\t iters: 60 / 70, epoch: 36 | loss: 0.0240749\n",
      "\t speed: 0.0215s/iter; left time: 96.6272s\n",
      "\t iters: 70 / 70, epoch: 36 | loss: 0.0207766\n",
      "\t speed: 0.0211s/iter; left time: 94.3858s\n",
      "Validation loss descreased: 0.017446365828315418 -> 0.017218281825383503\n",
      "Epoch: 36 Time: 1.5652692317962646 Steps: 70\n",
      "Train Loss: 0.0182047 | Val Loss: 0.0172183\n",
      "\t iters: 10 / 70, epoch: 37 | loss: 0.0164457\n",
      "\t speed: 0.0203s/iter; left time: 90.9834s\n",
      "\t iters: 20 / 70, epoch: 37 | loss: 0.0205105\n",
      "\t speed: 0.0219s/iter; left time: 97.6713s\n",
      "\t iters: 30 / 70, epoch: 37 | loss: 0.0143927\n",
      "\t speed: 0.0221s/iter; left time: 98.1780s\n",
      "\t iters: 40 / 70, epoch: 37 | loss: 0.0215371\n",
      "\t speed: 0.0214s/iter; left time: 94.9310s\n",
      "\t iters: 50 / 70, epoch: 37 | loss: 0.0215101\n",
      "\t speed: 0.0214s/iter; left time: 94.7532s\n",
      "\t iters: 60 / 70, epoch: 37 | loss: 0.0204413\n",
      "\t speed: 0.0213s/iter; left time: 94.0223s\n",
      "\t iters: 70 / 70, epoch: 37 | loss: 0.0178292\n",
      "\t speed: 0.0235s/iter; left time: 103.6887s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 37 Time: 1.557734489440918 Steps: 70\n",
      "Train Loss: 0.0182562 | Val Loss: 0.0181482\n",
      "\t iters: 10 / 70, epoch: 38 | loss: 0.0183562\n",
      "\t speed: 0.0223s/iter; left time: 98.0179s\n",
      "\t iters: 20 / 70, epoch: 38 | loss: 0.0168426\n",
      "\t speed: 0.0231s/iter; left time: 101.4235s\n",
      "\t iters: 30 / 70, epoch: 38 | loss: 0.0197534\n",
      "\t speed: 0.0230s/iter; left time: 100.6330s\n",
      "\t iters: 40 / 70, epoch: 38 | loss: 0.0142435\n",
      "\t speed: 0.0230s/iter; left time: 100.4529s\n",
      "\t iters: 50 / 70, epoch: 38 | loss: 0.0190372\n",
      "\t speed: 0.0236s/iter; left time: 102.8359s\n",
      "\t iters: 60 / 70, epoch: 38 | loss: 0.0166744\n",
      "\t speed: 0.0230s/iter; left time: 100.2051s\n",
      "\t iters: 70 / 70, epoch: 38 | loss: 0.0202950\n",
      "\t speed: 0.0230s/iter; left time: 99.7512s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 38 Time: 1.648266315460205 Steps: 70\n",
      "Train Loss: 0.0181862 | Val Loss: 0.0180473\n",
      "\t iters: 10 / 70, epoch: 39 | loss: 0.0205627\n",
      "\t speed: 0.0222s/iter; left time: 96.1081s\n",
      "\t iters: 20 / 70, epoch: 39 | loss: 0.0194063\n",
      "\t speed: 0.0235s/iter; left time: 101.6577s\n",
      "\t iters: 30 / 70, epoch: 39 | loss: 0.0211000\n",
      "\t speed: 0.0235s/iter; left time: 101.4598s\n",
      "\t iters: 40 / 70, epoch: 39 | loss: 0.0185781\n",
      "\t speed: 0.0233s/iter; left time: 100.3723s\n",
      "\t iters: 50 / 70, epoch: 39 | loss: 0.0181813\n",
      "\t speed: 0.0235s/iter; left time: 100.9909s\n",
      "\t iters: 60 / 70, epoch: 39 | loss: 0.0178416\n",
      "\t speed: 0.0236s/iter; left time: 101.2126s\n",
      "\t iters: 70 / 70, epoch: 39 | loss: 0.0162539\n",
      "\t speed: 0.0235s/iter; left time: 100.5055s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 39 Time: 1.6741950511932373 Steps: 70\n",
      "Train Loss: 0.0182007 | Val Loss: 0.0185604\n",
      "\t iters: 10 / 70, epoch: 40 | loss: 0.0160296\n",
      "\t speed: 0.0223s/iter; left time: 95.1612s\n",
      "\t iters: 20 / 70, epoch: 40 | loss: 0.0203175\n",
      "\t speed: 0.0236s/iter; left time: 100.4271s\n",
      "\t iters: 30 / 70, epoch: 40 | loss: 0.0169047\n",
      "\t speed: 0.0236s/iter; left time: 100.0909s\n",
      "\t iters: 40 / 70, epoch: 40 | loss: 0.0195141\n",
      "\t speed: 0.0232s/iter; left time: 98.0982s\n",
      "\t iters: 50 / 70, epoch: 40 | loss: 0.0152255\n",
      "\t speed: 0.0234s/iter; left time: 98.8981s\n",
      "\t iters: 60 / 70, epoch: 40 | loss: 0.0197054\n",
      "\t speed: 0.0232s/iter; left time: 97.6803s\n",
      "\t iters: 70 / 70, epoch: 40 | loss: 0.0190430\n",
      "\t speed: 0.0230s/iter; left time: 96.7177s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 40 Time: 1.6640284061431885 Steps: 70\n",
      "Train Loss: 0.0180771 | Val Loss: 0.0187025\n",
      "\t iters: 10 / 70, epoch: 41 | loss: 0.0196535\n",
      "\t speed: 0.0197s/iter; left time: 82.4194s\n",
      "\t iters: 20 / 70, epoch: 41 | loss: 0.0181455\n",
      "\t speed: 0.0206s/iter; left time: 86.1175s\n",
      "\t iters: 30 / 70, epoch: 41 | loss: 0.0171907\n",
      "\t speed: 0.0208s/iter; left time: 86.8485s\n",
      "\t iters: 40 / 70, epoch: 41 | loss: 0.0192433\n",
      "\t speed: 0.0212s/iter; left time: 88.2827s\n",
      "\t iters: 50 / 70, epoch: 41 | loss: 0.0184084\n",
      "\t speed: 0.0212s/iter; left time: 88.1607s\n",
      "\t iters: 60 / 70, epoch: 41 | loss: 0.0171934\n",
      "\t speed: 0.0211s/iter; left time: 87.3414s\n",
      "\t iters: 70 / 70, epoch: 41 | loss: 0.0207652\n",
      "\t speed: 0.0221s/iter; left time: 91.3337s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 41 Time: 1.5055336952209473 Steps: 70\n",
      "Train Loss: 0.0182311 | Val Loss: 0.0183160\n",
      "\t iters: 10 / 70, epoch: 42 | loss: 0.0196887\n",
      "\t speed: 0.0211s/iter; left time: 87.0738s\n",
      "\t iters: 20 / 70, epoch: 42 | loss: 0.0180570\n",
      "\t speed: 0.0217s/iter; left time: 89.2862s\n",
      "\t iters: 30 / 70, epoch: 42 | loss: 0.0173167\n",
      "\t speed: 0.0225s/iter; left time: 92.1813s\n",
      "\t iters: 40 / 70, epoch: 42 | loss: 0.0127158\n",
      "\t speed: 0.0224s/iter; left time: 91.6410s\n",
      "\t iters: 50 / 70, epoch: 42 | loss: 0.0153979\n",
      "\t speed: 0.0223s/iter; left time: 90.8624s\n",
      "\t iters: 60 / 70, epoch: 42 | loss: 0.0171646\n",
      "\t speed: 0.0225s/iter; left time: 91.3988s\n",
      "\t iters: 70 / 70, epoch: 42 | loss: 0.0212413\n",
      "\t speed: 0.0221s/iter; left time: 89.8040s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 42 Time: 1.5799245834350586 Steps: 70\n",
      "Train Loss: 0.0181285 | Val Loss: 0.0175297\n",
      "\t iters: 10 / 70, epoch: 43 | loss: 0.0170155\n",
      "\t speed: 0.0200s/iter; left time: 81.0790s\n",
      "\t iters: 20 / 70, epoch: 43 | loss: 0.0193828\n",
      "\t speed: 0.0205s/iter; left time: 82.6591s\n",
      "\t iters: 30 / 70, epoch: 43 | loss: 0.0187642\n",
      "\t speed: 0.0204s/iter; left time: 82.4145s\n",
      "\t iters: 40 / 70, epoch: 43 | loss: 0.0224075\n",
      "\t speed: 0.0210s/iter; left time: 84.2521s\n",
      "\t iters: 50 / 70, epoch: 43 | loss: 0.0239503\n",
      "\t speed: 0.0213s/iter; left time: 85.5489s\n",
      "\t iters: 60 / 70, epoch: 43 | loss: 0.0168572\n",
      "\t speed: 0.0222s/iter; left time: 88.8119s\n",
      "\t iters: 70 / 70, epoch: 43 | loss: 0.0186394\n",
      "\t speed: 0.0222s/iter; left time: 88.7545s\n",
      "Validation loss descreased: 0.017218281825383503 -> 0.017113846726715565\n",
      "Epoch: 43 Time: 1.5276172161102295 Steps: 70\n",
      "Train Loss: 0.0181849 | Val Loss: 0.0171138\n",
      "\t iters: 10 / 70, epoch: 44 | loss: 0.0164009\n",
      "\t speed: 0.0195s/iter; left time: 77.4641s\n",
      "\t iters: 20 / 70, epoch: 44 | loss: 0.0167967\n",
      "\t speed: 0.0220s/iter; left time: 87.2300s\n",
      "\t iters: 30 / 70, epoch: 44 | loss: 0.0216107\n",
      "\t speed: 0.0212s/iter; left time: 83.9835s\n",
      "\t iters: 40 / 70, epoch: 44 | loss: 0.0189668\n",
      "\t speed: 0.0216s/iter; left time: 85.2964s\n",
      "\t iters: 50 / 70, epoch: 44 | loss: 0.0158333\n",
      "\t speed: 0.0215s/iter; left time: 84.6034s\n",
      "\t iters: 60 / 70, epoch: 44 | loss: 0.0174663\n",
      "\t speed: 0.0221s/iter; left time: 86.9799s\n",
      "\t iters: 70 / 70, epoch: 44 | loss: 0.0153091\n",
      "\t speed: 0.0220s/iter; left time: 86.1983s\n",
      "Validation loss descreased: 0.017113846726715565 -> 0.016880581776301067\n",
      "Epoch: 44 Time: 1.546311616897583 Steps: 70\n",
      "Train Loss: 0.0180980 | Val Loss: 0.0168806\n",
      "\t iters: 10 / 70, epoch: 45 | loss: 0.0154303\n",
      "\t speed: 0.0200s/iter; left time: 78.1638s\n",
      "\t iters: 20 / 70, epoch: 45 | loss: 0.0202580\n",
      "\t speed: 0.0213s/iter; left time: 83.0953s\n",
      "\t iters: 30 / 70, epoch: 45 | loss: 0.0182658\n",
      "\t speed: 0.0222s/iter; left time: 86.2872s\n",
      "\t iters: 40 / 70, epoch: 45 | loss: 0.0181900\n",
      "\t speed: 0.0211s/iter; left time: 81.9279s\n",
      "\t iters: 50 / 70, epoch: 45 | loss: 0.0181221\n",
      "\t speed: 0.0233s/iter; left time: 90.3552s\n",
      "\t iters: 60 / 70, epoch: 45 | loss: 0.0164590\n",
      "\t speed: 0.0235s/iter; left time: 90.6552s\n",
      "\t iters: 70 / 70, epoch: 45 | loss: 0.0167040\n",
      "\t speed: 0.0220s/iter; left time: 84.6849s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 45 Time: 1.5692663192749023 Steps: 70\n",
      "Train Loss: 0.0181481 | Val Loss: 0.0174786\n",
      "\t iters: 10 / 70, epoch: 46 | loss: 0.0224165\n",
      "\t speed: 0.0198s/iter; left time: 76.1506s\n",
      "\t iters: 20 / 70, epoch: 46 | loss: 0.0181262\n",
      "\t speed: 0.0218s/iter; left time: 83.5603s\n",
      "\t iters: 30 / 70, epoch: 46 | loss: 0.0190766\n",
      "\t speed: 0.0219s/iter; left time: 83.8104s\n",
      "\t iters: 40 / 70, epoch: 46 | loss: 0.0179459\n",
      "\t speed: 0.0222s/iter; left time: 84.6538s\n",
      "\t iters: 50 / 70, epoch: 46 | loss: 0.0195804\n",
      "\t speed: 0.0220s/iter; left time: 83.6250s\n",
      "\t iters: 60 / 70, epoch: 46 | loss: 0.0179991\n",
      "\t speed: 0.0225s/iter; left time: 85.3177s\n",
      "\t iters: 70 / 70, epoch: 46 | loss: 0.0205859\n",
      "\t speed: 0.0208s/iter; left time: 78.5854s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 46 Time: 1.546281099319458 Steps: 70\n",
      "Train Loss: 0.0179984 | Val Loss: 0.0173122\n",
      "\t iters: 10 / 70, epoch: 47 | loss: 0.0178071\n",
      "\t speed: 0.0198s/iter; left time: 74.7733s\n",
      "\t iters: 20 / 70, epoch: 47 | loss: 0.0164203\n",
      "\t speed: 0.0208s/iter; left time: 78.0637s\n",
      "\t iters: 30 / 70, epoch: 47 | loss: 0.0188795\n",
      "\t speed: 0.0209s/iter; left time: 78.3690s\n",
      "\t iters: 40 / 70, epoch: 47 | loss: 0.0171864\n",
      "\t speed: 0.0214s/iter; left time: 80.0147s\n",
      "\t iters: 50 / 70, epoch: 47 | loss: 0.0164893\n",
      "\t speed: 0.0235s/iter; left time: 87.6162s\n",
      "\t iters: 60 / 70, epoch: 47 | loss: 0.0153696\n",
      "\t speed: 0.0233s/iter; left time: 86.8147s\n",
      "\t iters: 70 / 70, epoch: 47 | loss: 0.0214248\n",
      "\t speed: 0.0231s/iter; left time: 85.7391s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 47 Time: 1.566859483718872 Steps: 70\n",
      "Train Loss: 0.0180442 | Val Loss: 0.0181520\n",
      "\t iters: 10 / 70, epoch: 48 | loss: 0.0204028\n",
      "\t speed: 0.0209s/iter; left time: 77.4104s\n",
      "\t iters: 20 / 70, epoch: 48 | loss: 0.0141390\n",
      "\t speed: 0.0212s/iter; left time: 78.3577s\n",
      "\t iters: 30 / 70, epoch: 48 | loss: 0.0149107\n",
      "\t speed: 0.0211s/iter; left time: 77.7110s\n",
      "\t iters: 40 / 70, epoch: 48 | loss: 0.0218118\n",
      "\t speed: 0.0220s/iter; left time: 80.6521s\n",
      "\t iters: 50 / 70, epoch: 48 | loss: 0.0214384\n",
      "\t speed: 0.0208s/iter; left time: 76.1220s\n",
      "\t iters: 60 / 70, epoch: 48 | loss: 0.0185425\n",
      "\t speed: 0.0215s/iter; left time: 78.5110s\n",
      "\t iters: 70 / 70, epoch: 48 | loss: 0.0159413\n",
      "\t speed: 0.0223s/iter; left time: 81.0494s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 48 Time: 1.5359971523284912 Steps: 70\n",
      "Train Loss: 0.0180668 | Val Loss: 0.0179631\n",
      "\t iters: 10 / 70, epoch: 49 | loss: 0.0179165\n",
      "\t speed: 0.0210s/iter; left time: 76.2368s\n",
      "\t iters: 20 / 70, epoch: 49 | loss: 0.0170690\n",
      "\t speed: 0.0212s/iter; left time: 76.8821s\n",
      "\t iters: 30 / 70, epoch: 49 | loss: 0.0205753\n",
      "\t speed: 0.0213s/iter; left time: 77.0201s\n",
      "\t iters: 40 / 70, epoch: 49 | loss: 0.0165655\n",
      "\t speed: 0.0220s/iter; left time: 79.0709s\n",
      "\t iters: 50 / 70, epoch: 49 | loss: 0.0213986\n",
      "\t speed: 0.0220s/iter; left time: 79.0298s\n",
      "\t iters: 60 / 70, epoch: 49 | loss: 0.0159411\n",
      "\t speed: 0.0243s/iter; left time: 87.0076s\n",
      "\t iters: 70 / 70, epoch: 49 | loss: 0.0188005\n",
      "\t speed: 0.0243s/iter; left time: 86.7991s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 49 Time: 1.6022098064422607 Steps: 70\n",
      "Train Loss: 0.0180341 | Val Loss: 0.0179227\n",
      "\t iters: 10 / 70, epoch: 50 | loss: 0.0173712\n",
      "\t speed: 0.0210s/iter; left time: 74.6886s\n",
      "\t iters: 20 / 70, epoch: 50 | loss: 0.0179863\n",
      "\t speed: 0.0211s/iter; left time: 74.9629s\n",
      "\t iters: 30 / 70, epoch: 50 | loss: 0.0171211\n",
      "\t speed: 0.0206s/iter; left time: 72.9897s\n",
      "\t iters: 40 / 70, epoch: 50 | loss: 0.0195827\n",
      "\t speed: 0.0207s/iter; left time: 73.2535s\n",
      "\t iters: 50 / 70, epoch: 50 | loss: 0.0184097\n",
      "\t speed: 0.0211s/iter; left time: 74.3970s\n",
      "\t iters: 60 / 70, epoch: 50 | loss: 0.0128851\n",
      "\t speed: 0.0241s/iter; left time: 84.7592s\n",
      "\t iters: 70 / 70, epoch: 50 | loss: 0.0194506\n",
      "\t speed: 0.0240s/iter; left time: 84.0360s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 50 Time: 1.5622804164886475 Steps: 70\n",
      "Train Loss: 0.0180702 | Val Loss: 0.0181379\n",
      "\t iters: 10 / 70, epoch: 51 | loss: 0.0165871\n",
      "\t speed: 0.0201s/iter; left time: 70.3089s\n",
      "\t iters: 20 / 70, epoch: 51 | loss: 0.0155168\n",
      "\t speed: 0.0212s/iter; left time: 73.9383s\n",
      "\t iters: 30 / 70, epoch: 51 | loss: 0.0203698\n",
      "\t speed: 0.0215s/iter; left time: 74.5446s\n",
      "\t iters: 40 / 70, epoch: 51 | loss: 0.0189188\n",
      "\t speed: 0.0219s/iter; left time: 75.8934s\n",
      "\t iters: 50 / 70, epoch: 51 | loss: 0.0169273\n",
      "\t speed: 0.0219s/iter; left time: 75.4794s\n",
      "\t iters: 60 / 70, epoch: 51 | loss: 0.0182947\n",
      "\t speed: 0.0233s/iter; left time: 80.1487s\n",
      "\t iters: 70 / 70, epoch: 51 | loss: 0.0169036\n",
      "\t speed: 0.0220s/iter; left time: 75.3280s\n",
      "Validation loss did not decrease 7 / 20\n",
      "Epoch: 51 Time: 1.5596122741699219 Steps: 70\n",
      "Train Loss: 0.0179811 | Val Loss: 0.0174173\n",
      "\t iters: 10 / 70, epoch: 52 | loss: 0.0210356\n",
      "\t speed: 0.0229s/iter; left time: 78.5088s\n",
      "\t iters: 20 / 70, epoch: 52 | loss: 0.0187474\n",
      "\t speed: 0.0232s/iter; left time: 79.0516s\n",
      "\t iters: 30 / 70, epoch: 52 | loss: 0.0237686\n",
      "\t speed: 0.0215s/iter; left time: 72.9956s\n",
      "\t iters: 40 / 70, epoch: 52 | loss: 0.0185016\n",
      "\t speed: 0.0215s/iter; left time: 72.8619s\n",
      "\t iters: 50 / 70, epoch: 52 | loss: 0.0211892\n",
      "\t speed: 0.0211s/iter; left time: 71.2269s\n",
      "\t iters: 60 / 70, epoch: 52 | loss: 0.0140196\n",
      "\t speed: 0.0220s/iter; left time: 74.3111s\n",
      "\t iters: 70 / 70, epoch: 52 | loss: 0.0184652\n",
      "\t speed: 0.0219s/iter; left time: 73.4625s\n",
      "Validation loss did not decrease 8 / 20\n",
      "Epoch: 52 Time: 1.5761282444000244 Steps: 70\n",
      "Train Loss: 0.0180544 | Val Loss: 0.0182028\n",
      "\t iters: 10 / 70, epoch: 53 | loss: 0.0214374\n",
      "\t speed: 0.0206s/iter; left time: 68.8848s\n",
      "\t iters: 20 / 70, epoch: 53 | loss: 0.0172375\n",
      "\t speed: 0.0225s/iter; left time: 75.3356s\n",
      "\t iters: 30 / 70, epoch: 53 | loss: 0.0219447\n",
      "\t speed: 0.0216s/iter; left time: 71.8442s\n",
      "\t iters: 40 / 70, epoch: 53 | loss: 0.0168424\n",
      "\t speed: 0.0213s/iter; left time: 70.6443s\n",
      "\t iters: 50 / 70, epoch: 53 | loss: 0.0161434\n",
      "\t speed: 0.0214s/iter; left time: 70.9259s\n",
      "\t iters: 60 / 70, epoch: 53 | loss: 0.0170517\n",
      "\t speed: 0.0216s/iter; left time: 71.3657s\n",
      "\t iters: 70 / 70, epoch: 53 | loss: 0.0164535\n",
      "\t speed: 0.0220s/iter; left time: 72.2944s\n",
      "Validation loss did not decrease 9 / 20\n",
      "Epoch: 53 Time: 1.5446062088012695 Steps: 70\n",
      "Train Loss: 0.0179255 | Val Loss: 0.0186192\n",
      "\t iters: 10 / 70, epoch: 54 | loss: 0.0157746\n",
      "\t speed: 0.0208s/iter; left time: 68.2709s\n",
      "\t iters: 20 / 70, epoch: 54 | loss: 0.0170502\n",
      "\t speed: 0.0216s/iter; left time: 70.6635s\n",
      "\t iters: 30 / 70, epoch: 54 | loss: 0.0211402\n",
      "\t speed: 0.0223s/iter; left time: 72.8776s\n",
      "\t iters: 40 / 70, epoch: 54 | loss: 0.0160053\n",
      "\t speed: 0.0216s/iter; left time: 70.0995s\n",
      "\t iters: 50 / 70, epoch: 54 | loss: 0.0208181\n",
      "\t speed: 0.0216s/iter; left time: 69.9943s\n",
      "\t iters: 60 / 70, epoch: 54 | loss: 0.0156306\n",
      "\t speed: 0.0217s/iter; left time: 70.0411s\n",
      "\t iters: 70 / 70, epoch: 54 | loss: 0.0175220\n",
      "\t speed: 0.0220s/iter; left time: 70.7312s\n",
      "Validation loss did not decrease 10 / 20\n",
      "Epoch: 54 Time: 1.5524890422821045 Steps: 70\n",
      "Train Loss: 0.0178796 | Val Loss: 0.0186486\n",
      "\t iters: 10 / 70, epoch: 55 | loss: 0.0224950\n",
      "\t speed: 0.0203s/iter; left time: 65.2289s\n",
      "\t iters: 20 / 70, epoch: 55 | loss: 0.0153586\n",
      "\t speed: 0.0222s/iter; left time: 71.1406s\n",
      "\t iters: 30 / 70, epoch: 55 | loss: 0.0209210\n",
      "\t speed: 0.0217s/iter; left time: 69.1030s\n",
      "\t iters: 40 / 70, epoch: 55 | loss: 0.0170988\n",
      "\t speed: 0.0217s/iter; left time: 69.1293s\n",
      "\t iters: 50 / 70, epoch: 55 | loss: 0.0156016\n",
      "\t speed: 0.0221s/iter; left time: 69.9829s\n",
      "\t iters: 60 / 70, epoch: 55 | loss: 0.0168883\n",
      "\t speed: 0.0218s/iter; left time: 68.8979s\n",
      "\t iters: 70 / 70, epoch: 55 | loss: 0.0200328\n",
      "\t speed: 0.0223s/iter; left time: 70.2719s\n",
      "Validation loss did not decrease 11 / 20\n",
      "Epoch: 55 Time: 1.5560951232910156 Steps: 70\n",
      "Train Loss: 0.0180311 | Val Loss: 0.0187044\n",
      "\t iters: 10 / 70, epoch: 56 | loss: 0.0191341\n",
      "\t speed: 0.0205s/iter; left time: 64.4140s\n",
      "\t iters: 20 / 70, epoch: 56 | loss: 0.0168961\n",
      "\t speed: 0.0223s/iter; left time: 69.6836s\n",
      "\t iters: 30 / 70, epoch: 56 | loss: 0.0162036\n",
      "\t speed: 0.0218s/iter; left time: 67.9382s\n",
      "\t iters: 40 / 70, epoch: 56 | loss: 0.0198210\n",
      "\t speed: 0.0216s/iter; left time: 67.2465s\n",
      "\t iters: 50 / 70, epoch: 56 | loss: 0.0187833\n",
      "\t speed: 0.0210s/iter; left time: 65.2151s\n",
      "\t iters: 60 / 70, epoch: 56 | loss: 0.0195029\n",
      "\t speed: 0.0213s/iter; left time: 65.7895s\n",
      "\t iters: 70 / 70, epoch: 56 | loss: 0.0179452\n",
      "\t speed: 0.0215s/iter; left time: 66.3306s\n",
      "Validation loss did not decrease 12 / 20\n",
      "Epoch: 56 Time: 1.5348408222198486 Steps: 70\n",
      "Train Loss: 0.0179545 | Val Loss: 0.0182337\n",
      "\t iters: 10 / 70, epoch: 57 | loss: 0.0163463\n",
      "\t speed: 0.0202s/iter; left time: 62.1102s\n",
      "\t iters: 20 / 70, epoch: 57 | loss: 0.0154734\n",
      "\t speed: 0.0211s/iter; left time: 64.5858s\n",
      "\t iters: 30 / 70, epoch: 57 | loss: 0.0208561\n",
      "\t speed: 0.0211s/iter; left time: 64.2462s\n",
      "\t iters: 40 / 70, epoch: 57 | loss: 0.0168166\n",
      "\t speed: 0.0213s/iter; left time: 64.7524s\n",
      "\t iters: 50 / 70, epoch: 57 | loss: 0.0172269\n",
      "\t speed: 0.0213s/iter; left time: 64.5774s\n",
      "\t iters: 60 / 70, epoch: 57 | loss: 0.0200694\n",
      "\t speed: 0.0221s/iter; left time: 66.6746s\n",
      "\t iters: 70 / 70, epoch: 57 | loss: 0.0127784\n",
      "\t speed: 0.0223s/iter; left time: 67.0326s\n",
      "Validation loss descreased: 0.016880581776301067 -> 0.016580959782004356\n",
      "Epoch: 57 Time: 1.5440690517425537 Steps: 70\n",
      "Train Loss: 0.0179861 | Val Loss: 0.0165810\n",
      "\t iters: 10 / 70, epoch: 58 | loss: 0.0168711\n",
      "\t speed: 0.0212s/iter; left time: 63.6047s\n",
      "\t iters: 20 / 70, epoch: 58 | loss: 0.0135400\n",
      "\t speed: 0.0222s/iter; left time: 66.3896s\n",
      "\t iters: 30 / 70, epoch: 58 | loss: 0.0183718\n",
      "\t speed: 0.0221s/iter; left time: 65.7659s\n",
      "\t iters: 40 / 70, epoch: 58 | loss: 0.0191757\n",
      "\t speed: 0.0221s/iter; left time: 65.6924s\n",
      "\t iters: 50 / 70, epoch: 58 | loss: 0.0165994\n",
      "\t speed: 0.0222s/iter; left time: 65.8430s\n",
      "\t iters: 60 / 70, epoch: 58 | loss: 0.0177882\n",
      "\t speed: 0.0223s/iter; left time: 65.8735s\n",
      "\t iters: 70 / 70, epoch: 58 | loss: 0.0186976\n",
      "\t speed: 0.0215s/iter; left time: 63.1068s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 58 Time: 1.570396900177002 Steps: 70\n",
      "Train Loss: 0.0179276 | Val Loss: 0.0179573\n",
      "\t iters: 10 / 70, epoch: 59 | loss: 0.0157374\n",
      "\t speed: 0.0197s/iter; left time: 57.8142s\n",
      "\t iters: 20 / 70, epoch: 59 | loss: 0.0183846\n",
      "\t speed: 0.0217s/iter; left time: 63.4612s\n",
      "\t iters: 30 / 70, epoch: 59 | loss: 0.0172530\n",
      "\t speed: 0.0221s/iter; left time: 64.4045s\n",
      "\t iters: 40 / 70, epoch: 59 | loss: 0.0183799\n",
      "\t speed: 0.0221s/iter; left time: 64.1505s\n",
      "\t iters: 50 / 70, epoch: 59 | loss: 0.0177082\n",
      "\t speed: 0.0214s/iter; left time: 61.7901s\n",
      "\t iters: 60 / 70, epoch: 59 | loss: 0.0213483\n",
      "\t speed: 0.0223s/iter; left time: 64.1165s\n",
      "\t iters: 70 / 70, epoch: 59 | loss: 0.0197116\n",
      "\t speed: 0.0222s/iter; left time: 63.8447s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 59 Time: 1.5535433292388916 Steps: 70\n",
      "Train Loss: 0.0179848 | Val Loss: 0.0184694\n",
      "\t iters: 10 / 70, epoch: 60 | loss: 0.0194615\n",
      "\t speed: 0.0207s/iter; left time: 59.1494s\n",
      "\t iters: 20 / 70, epoch: 60 | loss: 0.0188745\n",
      "\t speed: 0.0214s/iter; left time: 61.1166s\n",
      "\t iters: 30 / 70, epoch: 60 | loss: 0.0181424\n",
      "\t speed: 0.0224s/iter; left time: 63.7636s\n",
      "\t iters: 40 / 70, epoch: 60 | loss: 0.0142569\n",
      "\t speed: 0.0220s/iter; left time: 62.2156s\n",
      "\t iters: 50 / 70, epoch: 60 | loss: 0.0166557\n",
      "\t speed: 0.0221s/iter; left time: 62.3059s\n",
      "\t iters: 60 / 70, epoch: 60 | loss: 0.0177073\n",
      "\t speed: 0.0213s/iter; left time: 59.9488s\n",
      "\t iters: 70 / 70, epoch: 60 | loss: 0.0165251\n",
      "\t speed: 0.0213s/iter; left time: 59.7643s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 60 Time: 1.551215648651123 Steps: 70\n",
      "Train Loss: 0.0179707 | Val Loss: 0.0171371\n",
      "\t iters: 10 / 70, epoch: 61 | loss: 0.0218005\n",
      "\t speed: 0.0209s/iter; left time: 58.3942s\n",
      "\t iters: 20 / 70, epoch: 61 | loss: 0.0204424\n",
      "\t speed: 0.0222s/iter; left time: 61.7306s\n",
      "\t iters: 30 / 70, epoch: 61 | loss: 0.0170889\n",
      "\t speed: 0.0222s/iter; left time: 61.4452s\n",
      "\t iters: 40 / 70, epoch: 61 | loss: 0.0131569\n",
      "\t speed: 0.0211s/iter; left time: 58.2368s\n",
      "\t iters: 50 / 70, epoch: 61 | loss: 0.0173452\n",
      "\t speed: 0.0225s/iter; left time: 61.9146s\n",
      "\t iters: 60 / 70, epoch: 61 | loss: 0.0179059\n",
      "\t speed: 0.0221s/iter; left time: 60.5167s\n",
      "\t iters: 70 / 70, epoch: 61 | loss: 0.0196899\n",
      "\t speed: 0.0218s/iter; left time: 59.6148s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 61 Time: 1.5628838539123535 Steps: 70\n",
      "Train Loss: 0.0179296 | Val Loss: 0.0172447\n",
      "\t iters: 10 / 70, epoch: 62 | loss: 0.0172398\n",
      "\t speed: 0.0203s/iter; left time: 55.1910s\n",
      "\t iters: 20 / 70, epoch: 62 | loss: 0.0167944\n",
      "\t speed: 0.0217s/iter; left time: 58.8871s\n",
      "\t iters: 30 / 70, epoch: 62 | loss: 0.0144883\n",
      "\t speed: 0.0222s/iter; left time: 60.0795s\n",
      "\t iters: 40 / 70, epoch: 62 | loss: 0.0152790\n",
      "\t speed: 0.0217s/iter; left time: 58.2612s\n",
      "\t iters: 50 / 70, epoch: 62 | loss: 0.0148110\n",
      "\t speed: 0.0226s/iter; left time: 60.5401s\n",
      "\t iters: 60 / 70, epoch: 62 | loss: 0.0173409\n",
      "\t speed: 0.0217s/iter; left time: 58.0547s\n",
      "\t iters: 70 / 70, epoch: 62 | loss: 0.0160774\n",
      "\t speed: 0.0223s/iter; left time: 59.2586s\n",
      "Validation loss descreased: 0.016580959782004356 -> 0.01647207699716091\n",
      "Epoch: 62 Time: 1.5761497020721436 Steps: 70\n",
      "Train Loss: 0.0179063 | Val Loss: 0.0164721\n",
      "\t iters: 10 / 70, epoch: 63 | loss: 0.0197881\n",
      "\t speed: 0.0211s/iter; left time: 55.8219s\n",
      "\t iters: 20 / 70, epoch: 63 | loss: 0.0187063\n",
      "\t speed: 0.0224s/iter; left time: 59.2080s\n",
      "\t iters: 30 / 70, epoch: 63 | loss: 0.0163757\n",
      "\t speed: 0.0225s/iter; left time: 59.1722s\n",
      "\t iters: 40 / 70, epoch: 63 | loss: 0.0201514\n",
      "\t speed: 0.0223s/iter; left time: 58.4734s\n",
      "\t iters: 50 / 70, epoch: 63 | loss: 0.0171142\n",
      "\t speed: 0.0224s/iter; left time: 58.5031s\n",
      "\t iters: 60 / 70, epoch: 63 | loss: 0.0216662\n",
      "\t speed: 0.0227s/iter; left time: 58.9871s\n",
      "\t iters: 70 / 70, epoch: 63 | loss: 0.0217445\n",
      "\t speed: 0.0220s/iter; left time: 57.1112s\n",
      "Validation loss did not decrease 1 / 20\n",
      "Epoch: 63 Time: 1.5921552181243896 Steps: 70\n",
      "Train Loss: 0.0179263 | Val Loss: 0.0179025\n",
      "\t iters: 10 / 70, epoch: 64 | loss: 0.0189754\n",
      "\t speed: 0.0209s/iter; left time: 53.8467s\n",
      "\t iters: 20 / 70, epoch: 64 | loss: 0.0168923\n",
      "\t speed: 0.0217s/iter; left time: 55.8766s\n",
      "\t iters: 30 / 70, epoch: 64 | loss: 0.0170154\n",
      "\t speed: 0.0223s/iter; left time: 57.0186s\n",
      "\t iters: 40 / 70, epoch: 64 | loss: 0.0195177\n",
      "\t speed: 0.0220s/iter; left time: 56.1937s\n",
      "\t iters: 50 / 70, epoch: 64 | loss: 0.0181391\n",
      "\t speed: 0.0218s/iter; left time: 55.3153s\n",
      "\t iters: 60 / 70, epoch: 64 | loss: 0.0158617\n",
      "\t speed: 0.0220s/iter; left time: 55.7014s\n",
      "\t iters: 70 / 70, epoch: 64 | loss: 0.0203758\n",
      "\t speed: 0.0218s/iter; left time: 54.9137s\n",
      "Validation loss did not decrease 2 / 20\n",
      "Epoch: 64 Time: 1.5619995594024658 Steps: 70\n",
      "Train Loss: 0.0179202 | Val Loss: 0.0186159\n",
      "\t iters: 10 / 70, epoch: 65 | loss: 0.0201811\n",
      "\t speed: 0.0213s/iter; left time: 53.5159s\n",
      "\t iters: 20 / 70, epoch: 65 | loss: 0.0192259\n",
      "\t speed: 0.0215s/iter; left time: 53.7888s\n",
      "\t iters: 30 / 70, epoch: 65 | loss: 0.0171332\n",
      "\t speed: 0.0227s/iter; left time: 56.5490s\n",
      "\t iters: 40 / 70, epoch: 65 | loss: 0.0194045\n",
      "\t speed: 0.0245s/iter; left time: 60.6606s\n",
      "\t iters: 50 / 70, epoch: 65 | loss: 0.0161033\n",
      "\t speed: 0.0221s/iter; left time: 54.4936s\n",
      "\t iters: 60 / 70, epoch: 65 | loss: 0.0178867\n",
      "\t speed: 0.0241s/iter; left time: 59.4008s\n",
      "\t iters: 70 / 70, epoch: 65 | loss: 0.0176269\n",
      "\t speed: 0.0233s/iter; left time: 57.1585s\n",
      "Validation loss did not decrease 3 / 20\n",
      "Epoch: 65 Time: 1.6361565589904785 Steps: 70\n",
      "Train Loss: 0.0179326 | Val Loss: 0.0185617\n",
      "\t iters: 10 / 70, epoch: 66 | loss: 0.0170429\n",
      "\t speed: 0.0218s/iter; left time: 53.1673s\n",
      "\t iters: 20 / 70, epoch: 66 | loss: 0.0145729\n",
      "\t speed: 0.0234s/iter; left time: 56.8110s\n",
      "\t iters: 30 / 70, epoch: 66 | loss: 0.0171200\n",
      "\t speed: 0.0235s/iter; left time: 57.0142s\n",
      "\t iters: 40 / 70, epoch: 66 | loss: 0.0200891\n",
      "\t speed: 0.0232s/iter; left time: 56.0381s\n",
      "\t iters: 50 / 70, epoch: 66 | loss: 0.0213909\n",
      "\t speed: 0.0230s/iter; left time: 55.3008s\n",
      "\t iters: 60 / 70, epoch: 66 | loss: 0.0137207\n",
      "\t speed: 0.0230s/iter; left time: 55.0371s\n",
      "\t iters: 70 / 70, epoch: 66 | loss: 0.0189597\n",
      "\t speed: 0.0230s/iter; left time: 54.7987s\n",
      "Validation loss did not decrease 4 / 20\n",
      "Epoch: 66 Time: 1.6512362957000732 Steps: 70\n",
      "Train Loss: 0.0178767 | Val Loss: 0.0166967\n",
      "\t iters: 10 / 70, epoch: 67 | loss: 0.0151582\n",
      "\t speed: 0.0215s/iter; left time: 51.0300s\n",
      "\t iters: 20 / 70, epoch: 67 | loss: 0.0179250\n",
      "\t speed: 0.0218s/iter; left time: 51.5138s\n",
      "\t iters: 30 / 70, epoch: 67 | loss: 0.0189948\n",
      "\t speed: 0.0217s/iter; left time: 51.0552s\n",
      "\t iters: 40 / 70, epoch: 67 | loss: 0.0167598\n",
      "\t speed: 0.0219s/iter; left time: 51.3201s\n",
      "\t iters: 50 / 70, epoch: 67 | loss: 0.0178698\n",
      "\t speed: 0.0217s/iter; left time: 50.6592s\n",
      "\t iters: 60 / 70, epoch: 67 | loss: 0.0181493\n",
      "\t speed: 0.0238s/iter; left time: 55.2044s\n",
      "\t iters: 70 / 70, epoch: 67 | loss: 0.0147692\n",
      "\t speed: 0.0230s/iter; left time: 53.2223s\n",
      "Validation loss did not decrease 5 / 20\n",
      "Epoch: 67 Time: 1.5896480083465576 Steps: 70\n",
      "Train Loss: 0.0179643 | Val Loss: 0.0174671\n",
      "\t iters: 10 / 70, epoch: 68 | loss: 0.0148887\n",
      "\t speed: 0.0199s/iter; left time: 45.8259s\n",
      "\t iters: 20 / 70, epoch: 68 | loss: 0.0211689\n",
      "\t speed: 0.0211s/iter; left time: 48.3960s\n",
      "\t iters: 30 / 70, epoch: 68 | loss: 0.0180174\n",
      "\t speed: 0.0211s/iter; left time: 48.1995s\n",
      "\t iters: 40 / 70, epoch: 68 | loss: 0.0156043\n",
      "\t speed: 0.0231s/iter; left time: 52.5091s\n",
      "\t iters: 50 / 70, epoch: 68 | loss: 0.0143460\n",
      "\t speed: 0.0232s/iter; left time: 52.4209s\n",
      "\t iters: 60 / 70, epoch: 68 | loss: 0.0171806\n",
      "\t speed: 0.0224s/iter; left time: 50.3814s\n",
      "\t iters: 70 / 70, epoch: 68 | loss: 0.0182948\n",
      "\t speed: 0.0229s/iter; left time: 51.3278s\n",
      "Validation loss did not decrease 6 / 20\n",
      "Epoch: 68 Time: 1.5768966674804688 Steps: 70\n",
      "Train Loss: 0.0177573 | Val Loss: 0.0175994\n",
      "\t iters: 10 / 70, epoch: 69 | loss: 0.0200627\n",
      "\t speed: 0.0219s/iter; left time: 48.8098s\n",
      "\t iters: 20 / 70, epoch: 69 | loss: 0.0169515\n",
      "\t speed: 0.0234s/iter; left time: 51.9395s\n",
      "\t iters: 30 / 70, epoch: 69 | loss: 0.0173223\n",
      "\t speed: 0.0232s/iter; left time: 51.3115s\n",
      "\t iters: 40 / 70, epoch: 69 | loss: 0.0156886\n",
      "\t speed: 0.0233s/iter; left time: 51.2754s\n",
      "\t iters: 50 / 70, epoch: 69 | loss: 0.0202373\n",
      "\t speed: 0.0234s/iter; left time: 51.2456s\n",
      "\t iters: 60 / 70, epoch: 69 | loss: 0.0174724\n",
      "\t speed: 0.0231s/iter; left time: 50.4258s\n",
      "\t iters: 70 / 70, epoch: 69 | loss: 0.0183781\n",
      "\t speed: 0.0235s/iter; left time: 50.9680s\n",
      "Validation loss did not decrease 7 / 20\n",
      "Epoch: 69 Time: 1.6579902172088623 Steps: 70\n",
      "Train Loss: 0.0179047 | Val Loss: 0.0185119\n",
      "\t iters: 10 / 70, epoch: 70 | loss: 0.0200639\n",
      "\t speed: 0.0225s/iter; left time: 48.6440s\n",
      "\t iters: 20 / 70, epoch: 70 | loss: 0.0167877\n",
      "\t speed: 0.0231s/iter; left time: 49.7166s\n",
      "\t iters: 30 / 70, epoch: 70 | loss: 0.0151007\n",
      "\t speed: 0.0230s/iter; left time: 49.2503s\n",
      "\t iters: 40 / 70, epoch: 70 | loss: 0.0191245\n",
      "\t speed: 0.0230s/iter; left time: 48.9728s\n",
      "\t iters: 50 / 70, epoch: 70 | loss: 0.0149790\n",
      "\t speed: 0.0233s/iter; left time: 49.4531s\n",
      "\t iters: 60 / 70, epoch: 70 | loss: 0.0138107\n",
      "\t speed: 0.0213s/iter; left time: 44.9380s\n",
      "\t iters: 70 / 70, epoch: 70 | loss: 0.0201448\n",
      "\t speed: 0.0216s/iter; left time: 45.4023s\n",
      "Validation loss did not decrease 8 / 20\n",
      "Epoch: 70 Time: 1.6134214401245117 Steps: 70\n",
      "Train Loss: 0.0179657 | Val Loss: 0.0187864\n",
      "\t iters: 10 / 70, epoch: 71 | loss: 0.0168023\n",
      "\t speed: 0.0207s/iter; left time: 43.2453s\n",
      "\t iters: 20 / 70, epoch: 71 | loss: 0.0163246\n",
      "\t speed: 0.0241s/iter; left time: 50.2250s\n",
      "\t iters: 30 / 70, epoch: 71 | loss: 0.0220034\n",
      "\t speed: 0.0243s/iter; left time: 50.4033s\n",
      "\t iters: 40 / 70, epoch: 71 | loss: 0.0171388\n",
      "\t speed: 0.0246s/iter; left time: 50.7840s\n",
      "\t iters: 50 / 70, epoch: 71 | loss: 0.0183102\n",
      "\t speed: 0.0215s/iter; left time: 43.9947s\n",
      "\t iters: 60 / 70, epoch: 71 | loss: 0.0173129\n",
      "\t speed: 0.0212s/iter; left time: 43.1919s\n",
      "\t iters: 70 / 70, epoch: 71 | loss: 0.0182629\n",
      "\t speed: 0.0214s/iter; left time: 43.3864s\n",
      "Validation loss did not decrease 9 / 20\n",
      "Epoch: 71 Time: 1.617983102798462 Steps: 70\n",
      "Train Loss: 0.0178803 | Val Loss: 0.0175084\n",
      "\t iters: 10 / 70, epoch: 72 | loss: 0.0187403\n",
      "\t speed: 0.0221s/iter; left time: 44.6130s\n",
      "\t iters: 20 / 70, epoch: 72 | loss: 0.0165614\n",
      "\t speed: 0.0236s/iter; left time: 47.4803s\n",
      "\t iters: 30 / 70, epoch: 72 | loss: 0.0150794\n",
      "\t speed: 0.0223s/iter; left time: 44.7091s\n",
      "\t iters: 40 / 70, epoch: 72 | loss: 0.0175216\n",
      "\t speed: 0.0225s/iter; left time: 44.6999s\n",
      "\t iters: 50 / 70, epoch: 72 | loss: 0.0169301\n",
      "\t speed: 0.0217s/iter; left time: 42.8897s\n",
      "\t iters: 60 / 70, epoch: 72 | loss: 0.0160930\n",
      "\t speed: 0.0219s/iter; left time: 43.2049s\n",
      "\t iters: 70 / 70, epoch: 72 | loss: 0.0165963\n",
      "\t speed: 0.0217s/iter; left time: 42.5392s\n",
      "Validation loss did not decrease 10 / 20\n",
      "Epoch: 72 Time: 1.594508171081543 Steps: 70\n",
      "Train Loss: 0.0178509 | Val Loss: 0.0179072\n",
      "\t iters: 10 / 70, epoch: 73 | loss: 0.0170110\n",
      "\t speed: 0.0199s/iter; left time: 38.8857s\n",
      "\t iters: 20 / 70, epoch: 73 | loss: 0.0188826\n",
      "\t speed: 0.0222s/iter; left time: 43.0420s\n",
      "\t iters: 30 / 70, epoch: 73 | loss: 0.0216920\n",
      "\t speed: 0.0242s/iter; left time: 46.7443s\n",
      "\t iters: 40 / 70, epoch: 73 | loss: 0.0162859\n",
      "\t speed: 0.0240s/iter; left time: 46.1797s\n",
      "\t iters: 50 / 70, epoch: 73 | loss: 0.0171629\n",
      "\t speed: 0.0209s/iter; left time: 39.9977s\n",
      "\t iters: 60 / 70, epoch: 73 | loss: 0.0230139\n",
      "\t speed: 0.0208s/iter; left time: 39.6185s\n",
      "\t iters: 70 / 70, epoch: 73 | loss: 0.0172181\n",
      "\t speed: 0.0210s/iter; left time: 39.6836s\n",
      "Validation loss did not decrease 11 / 20\n",
      "Epoch: 73 Time: 1.5661516189575195 Steps: 70\n",
      "Train Loss: 0.0178278 | Val Loss: 0.0172859\n",
      "\t iters: 10 / 70, epoch: 74 | loss: 0.0202758\n",
      "\t speed: 0.0198s/iter; left time: 37.1873s\n",
      "\t iters: 20 / 70, epoch: 74 | loss: 0.0221534\n",
      "\t speed: 0.0210s/iter; left time: 39.3788s\n",
      "\t iters: 30 / 70, epoch: 74 | loss: 0.0240485\n",
      "\t speed: 0.0211s/iter; left time: 39.3028s\n",
      "\t iters: 40 / 70, epoch: 74 | loss: 0.0158010\n",
      "\t speed: 0.0212s/iter; left time: 39.1593s\n",
      "\t iters: 50 / 70, epoch: 74 | loss: 0.0185759\n",
      "\t speed: 0.0213s/iter; left time: 39.2060s\n",
      "\t iters: 60 / 70, epoch: 74 | loss: 0.0199839\n",
      "\t speed: 0.0214s/iter; left time: 39.2308s\n",
      "\t iters: 70 / 70, epoch: 74 | loss: 0.0208363\n",
      "\t speed: 0.0217s/iter; left time: 39.5507s\n",
      "Validation loss did not decrease 12 / 20\n",
      "Epoch: 74 Time: 1.5127134323120117 Steps: 70\n",
      "Train Loss: 0.0177894 | Val Loss: 0.0174199\n",
      "\t iters: 10 / 70, epoch: 75 | loss: 0.0174747\n",
      "\t speed: 0.0211s/iter; left time: 38.1683s\n",
      "\t iters: 20 / 70, epoch: 75 | loss: 0.0186294\n",
      "\t speed: 0.0222s/iter; left time: 39.9240s\n",
      "\t iters: 30 / 70, epoch: 75 | loss: 0.0172850\n",
      "\t speed: 0.0214s/iter; left time: 38.3033s\n",
      "\t iters: 40 / 70, epoch: 75 | loss: 0.0149197\n",
      "\t speed: 0.0211s/iter; left time: 37.6520s\n",
      "\t iters: 50 / 70, epoch: 75 | loss: 0.0195091\n",
      "\t speed: 0.0217s/iter; left time: 38.4888s\n",
      "\t iters: 60 / 70, epoch: 75 | loss: 0.0172413\n",
      "\t speed: 0.0216s/iter; left time: 38.0337s\n",
      "\t iters: 70 / 70, epoch: 75 | loss: 0.0170517\n",
      "\t speed: 0.0213s/iter; left time: 37.3040s\n",
      "Validation loss did not decrease 13 / 20\n",
      "Epoch: 75 Time: 1.5400447845458984 Steps: 70\n",
      "Train Loss: 0.0178595 | Val Loss: 0.0180833\n",
      "\t iters: 10 / 70, epoch: 76 | loss: 0.0166641\n",
      "\t speed: 0.0201s/iter; left time: 35.0144s\n",
      "\t iters: 20 / 70, epoch: 76 | loss: 0.0186026\n",
      "\t speed: 0.0212s/iter; left time: 36.6392s\n",
      "\t iters: 30 / 70, epoch: 76 | loss: 0.0182191\n",
      "\t speed: 0.0213s/iter; left time: 36.6265s\n",
      "\t iters: 40 / 70, epoch: 76 | loss: 0.0170101\n",
      "\t speed: 0.0210s/iter; left time: 35.9315s\n",
      "\t iters: 50 / 70, epoch: 76 | loss: 0.0212783\n",
      "\t speed: 0.0220s/iter; left time: 37.4035s\n",
      "\t iters: 60 / 70, epoch: 76 | loss: 0.0177524\n",
      "\t speed: 0.0220s/iter; left time: 37.1263s\n",
      "\t iters: 70 / 70, epoch: 76 | loss: 0.0215322\n",
      "\t speed: 0.0217s/iter; left time: 36.4811s\n",
      "Validation loss did not decrease 14 / 20\n",
      "Epoch: 76 Time: 1.5274176597595215 Steps: 70\n",
      "Train Loss: 0.0178887 | Val Loss: 0.0172476\n",
      "\t iters: 10 / 70, epoch: 77 | loss: 0.0179201\n",
      "\t speed: 0.0204s/iter; left time: 34.0664s\n",
      "\t iters: 20 / 70, epoch: 77 | loss: 0.0181134\n",
      "\t speed: 0.0220s/iter; left time: 36.6017s\n",
      "\t iters: 30 / 70, epoch: 77 | loss: 0.0189740\n",
      "\t speed: 0.0220s/iter; left time: 36.3388s\n",
      "\t iters: 40 / 70, epoch: 77 | loss: 0.0166701\n",
      "\t speed: 0.0217s/iter; left time: 35.5458s\n",
      "\t iters: 50 / 70, epoch: 77 | loss: 0.0181603\n",
      "\t speed: 0.0219s/iter; left time: 35.7700s\n",
      "\t iters: 60 / 70, epoch: 77 | loss: 0.0182230\n",
      "\t speed: 0.0216s/iter; left time: 34.9516s\n",
      "\t iters: 70 / 70, epoch: 77 | loss: 0.0195715\n",
      "\t speed: 0.0214s/iter; left time: 34.5299s\n",
      "Validation loss did not decrease 15 / 20\n",
      "Epoch: 77 Time: 1.5479884147644043 Steps: 70\n",
      "Train Loss: 0.0178179 | Val Loss: 0.0178305\n",
      "\t iters: 10 / 70, epoch: 78 | loss: 0.0178852\n",
      "\t speed: 0.0202s/iter; left time: 32.3170s\n",
      "\t iters: 20 / 70, epoch: 78 | loss: 0.0218050\n",
      "\t speed: 0.0213s/iter; left time: 33.8169s\n",
      "\t iters: 30 / 70, epoch: 78 | loss: 0.0166203\n",
      "\t speed: 0.0221s/iter; left time: 34.9678s\n",
      "\t iters: 40 / 70, epoch: 78 | loss: 0.0208053\n",
      "\t speed: 0.0247s/iter; left time: 38.8610s\n",
      "\t iters: 50 / 70, epoch: 78 | loss: 0.0186627\n",
      "\t speed: 0.0223s/iter; left time: 34.7549s\n",
      "\t iters: 60 / 70, epoch: 78 | loss: 0.0185625\n",
      "\t speed: 0.0221s/iter; left time: 34.2175s\n",
      "\t iters: 70 / 70, epoch: 78 | loss: 0.0189090\n",
      "\t speed: 0.0226s/iter; left time: 34.8776s\n",
      "Validation loss did not decrease 16 / 20\n",
      "Epoch: 78 Time: 1.590956449508667 Steps: 70\n",
      "Train Loss: 0.0178197 | Val Loss: 0.0182775\n",
      "\t iters: 10 / 70, epoch: 79 | loss: 0.0172273\n",
      "\t speed: 0.0215s/iter; left time: 32.8806s\n",
      "\t iters: 20 / 70, epoch: 79 | loss: 0.0156373\n",
      "\t speed: 0.0227s/iter; left time: 34.5033s\n",
      "\t iters: 30 / 70, epoch: 79 | loss: 0.0200181\n",
      "\t speed: 0.0223s/iter; left time: 33.6868s\n",
      "\t iters: 40 / 70, epoch: 79 | loss: 0.0172282\n",
      "\t speed: 0.0221s/iter; left time: 33.2318s\n",
      "\t iters: 50 / 70, epoch: 79 | loss: 0.0190129\n",
      "\t speed: 0.0223s/iter; left time: 33.1866s\n",
      "\t iters: 60 / 70, epoch: 79 | loss: 0.0182034\n",
      "\t speed: 0.0220s/iter; left time: 32.6424s\n",
      "\t iters: 70 / 70, epoch: 79 | loss: 0.0173265\n",
      "\t speed: 0.0224s/iter; left time: 32.9029s\n",
      "Validation loss did not decrease 17 / 20\n",
      "Epoch: 79 Time: 1.5893244743347168 Steps: 70\n",
      "Train Loss: 0.0178433 | Val Loss: 0.0181104\n",
      "\t iters: 10 / 70, epoch: 80 | loss: 0.0185611\n",
      "\t speed: 0.0198s/iter; left time: 28.9077s\n",
      "\t iters: 20 / 70, epoch: 80 | loss: 0.0162338\n",
      "\t speed: 0.0235s/iter; left time: 34.1089s\n",
      "\t iters: 30 / 70, epoch: 80 | loss: 0.0197202\n",
      "\t speed: 0.0231s/iter; left time: 33.2739s\n",
      "\t iters: 40 / 70, epoch: 80 | loss: 0.0222146\n",
      "\t speed: 0.0228s/iter; left time: 32.5568s\n",
      "\t iters: 50 / 70, epoch: 80 | loss: 0.0185953\n",
      "\t speed: 0.0236s/iter; left time: 33.5969s\n",
      "\t iters: 60 / 70, epoch: 80 | loss: 0.0175770\n",
      "\t speed: 0.0230s/iter; left time: 32.3944s\n",
      "\t iters: 70 / 70, epoch: 80 | loss: 0.0156543\n",
      "\t speed: 0.0228s/iter; left time: 31.9597s\n",
      "Validation loss did not decrease 18 / 20\n",
      "Epoch: 80 Time: 1.6245901584625244 Steps: 70\n",
      "Train Loss: 0.0178706 | Val Loss: 0.0184832\n",
      "\t iters: 10 / 70, epoch: 81 | loss: 0.0180478\n",
      "\t speed: 0.0224s/iter; left time: 31.1629s\n",
      "\t iters: 20 / 70, epoch: 81 | loss: 0.0203150\n",
      "\t speed: 0.0229s/iter; left time: 31.6318s\n",
      "\t iters: 30 / 70, epoch: 81 | loss: 0.0144031\n",
      "\t speed: 0.0230s/iter; left time: 31.5685s\n",
      "\t iters: 40 / 70, epoch: 81 | loss: 0.0164197\n",
      "\t speed: 0.0237s/iter; left time: 32.2560s\n",
      "\t iters: 50 / 70, epoch: 81 | loss: 0.0159270\n",
      "\t speed: 0.0230s/iter; left time: 31.1046s\n",
      "\t iters: 60 / 70, epoch: 81 | loss: 0.0212463\n",
      "\t speed: 0.0236s/iter; left time: 31.6758s\n",
      "\t iters: 70 / 70, epoch: 81 | loss: 0.0160783\n",
      "\t speed: 0.0230s/iter; left time: 30.6114s\n",
      "Validation loss did not decrease 19 / 20\n",
      "Epoch: 81 Time: 1.6576790809631348 Steps: 70\n",
      "Train Loss: 0.0177655 | Val Loss: 0.0173217\n",
      "\t iters: 10 / 70, epoch: 82 | loss: 0.0197576\n",
      "\t speed: 0.0199s/iter; left time: 26.3447s\n",
      "\t iters: 20 / 70, epoch: 82 | loss: 0.0218509\n",
      "\t speed: 0.0211s/iter; left time: 27.7172s\n",
      "\t iters: 30 / 70, epoch: 82 | loss: 0.0178509\n",
      "\t speed: 0.0242s/iter; left time: 31.5238s\n",
      "\t iters: 40 / 70, epoch: 82 | loss: 0.0177920\n",
      "\t speed: 0.0240s/iter; left time: 30.9207s\n",
      "\t iters: 50 / 70, epoch: 82 | loss: 0.0165335\n",
      "\t speed: 0.0214s/iter; left time: 27.4012s\n",
      "\t iters: 60 / 70, epoch: 82 | loss: 0.0186306\n",
      "\t speed: 0.0216s/iter; left time: 27.4204s\n",
      "\t iters: 70 / 70, epoch: 82 | loss: 0.0141443\n",
      "\t speed: 0.0221s/iter; left time: 27.8995s\n",
      "Validation loss did not decrease 20 / 20\n",
      "Early stopping...\n"
     ]
    }
   ],
   "source": [
    "epochs = 100\n",
    "criterion = nn.MSELoss()\n",
    "model_optim = torch.optim.Adam(model.parameters(), lr=0.0002, betas=(0, 0.9))\n",
    "\n",
    "# define params for training\n",
    "TRAIN_STEPS = len(train_data_loader)\n",
    "early_stop = EarlyStop(20, 1e-6)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    iter_count = 0\n",
    "    train_loss = []\n",
    "    \n",
    "    epoch_time = time.time()\n",
    "    curr_time = time.time()\n",
    "    \n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(train_data_loader):\n",
    "        iter_count += 1\n",
    "        # zero-out grad\n",
    "        model_optim.zero_grad()\n",
    "        pred, true, logvar = process_batch(subj_id=subj_id,\n",
    "                                            batch_x=batch_x, \n",
    "                                            batch_y=batch_y, \n",
    "                                            batch_x_mark=batch_x_mark, \n",
    "                                            batch_y_mark=batch_y_mark, \n",
    "                                            len_pred=len_pred, \n",
    "                                            len_label=len_label, \n",
    "                                            model=model, \n",
    "                                            device=device)\n",
    "        loss = criterion(pred, true)\n",
    "        train_loss.append(float(loss.item()))\n",
    "        \n",
    "        # print every 10\n",
    "        if (i+1) % 10==0:\n",
    "            print(\"\\t iters: {0} / {3}, epoch: {1} | loss: {2:.7f}\".format(i + 1, epoch + 1, loss.item(), TRAIN_STEPS))\n",
    "            speed = (time.time() - curr_time) / iter_count\n",
    "            left_time = speed * ((epochs - epoch) * TRAIN_STEPS - i)\n",
    "            print('\\t speed: {:.4f}s/iter; left time: {:.4f}s'.format(speed, left_time))\n",
    "            iter_count = 0\n",
    "            curr_time = time.time()\n",
    "        \n",
    "        loss.backward()\n",
    "        model_optim.step()\n",
    "    # compute average train loss\n",
    "    train_loss = np.average(train_loss)\n",
    "\n",
    "    # compute validation / test loss + test metric\n",
    "    with torch.no_grad():\n",
    "        val_loss = []\n",
    "        for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(val_data_loader):\n",
    "            pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                    batch_x=batch_x, \n",
    "                                    batch_y=batch_y, \n",
    "                                    batch_x_mark=batch_x_mark, \n",
    "                                    batch_y_mark=batch_y_mark, \n",
    "                                    len_pred=len_pred, \n",
    "                                    len_label=len_label, \n",
    "                                    model=model, \n",
    "                                    device=device)\n",
    "            loss = criterion(pred, true)\n",
    "            val_loss.append(float(loss.item()))\n",
    "        val_loss = np.average(val_loss)\n",
    "    \n",
    "    # check early stopping\n",
    "    early_stop(val_loss, model, model_path)\n",
    "    if early_stop.stop:\n",
    "        print(\"Early stopping...\")\n",
    "        break\n",
    "\n",
    "    # update lr\n",
    "    # adjust_learning_rate(model_optim, epoch, lr)\n",
    "    \n",
    "    print(\"Epoch: {0} Time: {1} Steps: {2}\".format(epoch+1, time.time() - epoch_time, TRAIN_STEPS))\n",
    "    print(\"Train Loss: {0:.7f} | Val Loss: {1:.7f}\".format(train_loss, val_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test normal model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APE: 0.3268\n",
      "RMSE: 0.1415\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "ape, rmse, likelihood = [], [], []\n",
    "with torch.no_grad():\n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(test_data_loader):\n",
    "        pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                batch_x=batch_x, \n",
    "                                batch_y=batch_y, \n",
    "                                batch_x_mark=batch_x_mark, \n",
    "                                batch_y_mark=batch_y_mark, \n",
    "                                len_pred=len_pred, \n",
    "                                len_label=len_label, \n",
    "                                model=model, \n",
    "                                device=device)\n",
    "        \n",
    "        # arrange in proper shape: take mean of predicted samples\n",
    "        pred = pred.detach().cpu().numpy(); true = true.detach().cpu().numpy(); logvar = logvar.detach().cpu().numpy()\n",
    "        pred = pred.transpose((1,0,2)).reshape((pred.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        true = true.transpose((1,0,2)).reshape((true.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        # for metrics: take mean of smaples, take one sample of true\n",
    "        pred = np.mean(pred, axis=2)\n",
    "        true = true[:, :, 0]\n",
    "        # compute APE / RMSE\n",
    "        ape.append(np.mean(np.abs(true - pred) / true))\n",
    "        rmse.append(np.sqrt(np.mean((true - pred)**2)))\n",
    "        # compute likelihood for computing var later\n",
    "        likelihood.append(np.mean((pred - true)**2, axis=1))\n",
    "\n",
    "varhat = np.mean(np.concatenate(likelihood, axis=0))\n",
    "rmse = np.median(rmse)\n",
    "ape = np.median(ape)\n",
    "print(\"APE: {0:.4f}\".format(ape))\n",
    "print(\"RMSE: {0:.4f}\".format(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save for plotting\n",
    "with torch.no_grad():\n",
    "    for i, (subj_id, batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(test_data_loader):\n",
    "        pred, true, logvar = process_batch(subj_id = subj_id, \n",
    "                                batch_x=batch_x, \n",
    "                                batch_y=batch_y, \n",
    "                                batch_x_mark=batch_x_mark, \n",
    "                                batch_y_mark=batch_y_mark, \n",
    "                                len_pred=len_pred, \n",
    "                                len_label=len_label, \n",
    "                                model=model, \n",
    "                                device=device)\n",
    "        \n",
    "        # arrange in proper shape: take mean of predicted samples\n",
    "        pred = pred.detach().cpu().numpy(); true = true.detach().cpu().numpy()\n",
    "        batch_x = batch_x.detach().cpu().numpy(); batch_x_mark = batch_x_mark.detach().cpu().numpy()\n",
    "        pred = pred.transpose((1,0,2)).reshape((pred.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        true = true.transpose((1,0,2)).reshape((true.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        batch_x = batch_x.transpose((1,0,2)).reshape((batch_x.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        batch_x_mark = batch_x_mark.transpose((1,0,2)).reshape((batch_x_mark.shape[1], -1, num_samples)).transpose((1, 0, 2))\n",
    "        break\n",
    "# save\n",
    "np.save(f\"{cache_path}/pred_mean_norm.npy\", pred)\n",
    "np.save(f\"{cache_path}/true_mean_norm.npy\", true)\n",
    "np.save(f\"{cache_path}/pred_var_norm.npy\", np.array([varhat]))\n",
    "np.save(f\"{cache_path}/input_norm.npy\", batch_x)\n",
    "np.save(f\"{cache_path}/input_x_norm.npy\", batch_x_mark)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = (8, *[1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = set(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "285382b90770c3e479aa722115be92e757ae824c82ed0278c4e38e83bd68deda"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('gluformer')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}