tranformer-eng-to-urdu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:58:47.250339Z",
     "iopub.status.busy": "2024-11-23T11:58:47.249819Z",
     "iopub.status.idle": "2024-11-23T11:58:57.536148Z",
     "shell.execute_reply": "2024-11-23T11:58:57.534999Z",
     "shell.execute_reply.started": "2024-11-23T11:58:47.250290Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "! pip install --q evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:58:57.538831Z",
     "iopub.status.busy": "2024-11-23T11:58:57.538419Z",
     "iopub.status.idle": "2024-11-23T11:59:11.380571Z",
     "shell.execute_reply": "2024-11-23T11:59:11.379631Z",
     "shell.execute_reply.started": "2024-11-23T11:58:57.538795Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-23 11:59:01.778564: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-11-23 11:59:01.778719: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-11-23 11:59:01.905721: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import torch\n",
    "import evaluate\n",
    "\n",
    "from torch import nn\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import Tokenizer\n",
    "from tokenizers import normalizers\n",
    "from tokenizers import pre_tokenizers\n",
    "from tokenizers.models import WordLevel, WordPiece\n",
    "from tokenizers.processors import TemplateProcessing\n",
    "\n",
    "\n",
    "from torch.utils.data import DataLoader, Dataset, Subset\n",
    "\n",
    "from tokenizers.normalizers import NFC, StripAccents, Strip, Lowercase\n",
    "from tokenizers.pre_tokenizers import Whitespace\n",
    "from tokenizers.trainers import WordLevelTrainer, WordPieceTrainer\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:55.950372Z",
     "iopub.status.busy": "2024-11-23T11:59:55.949433Z",
     "iopub.status.idle": "2024-11-23T11:59:55.955559Z",
     "shell.execute_reply": "2024-11-23T11:59:55.954560Z",
     "shell.execute_reply.started": "2024-11-23T11:59:55.950342Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : Class of myParameters\n",
    "\n",
    "class Variables :\n",
    "    TEXT ={\n",
    "        #'vocab_size' : 200_000,\n",
    "        'seq_len':20,\n",
    "        'batch':32\n",
    "    }\n",
    "    TRANSFORMER = {\n",
    "        'heads':16,\n",
    "        'latent_dim':512,\n",
    "        'pf_dim':1024,\n",
    "        'dropout':0.25,\n",
    "        'encoder_layers':3,\n",
    "        'decoder_layers':3,\n",
    "    }\n",
    "    TRAIN = {\n",
    "    'epoch'  : 500,\n",
    "    'lr'     : 1e-4,\n",
    "    }\n",
    "    \n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    \n",
    "var = Variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:56.146108Z",
     "iopub.status.busy": "2024-11-23T11:59:56.145587Z",
     "iopub.status.idle": "2024-11-23T11:59:56.152116Z",
     "shell.execute_reply": "2024-11-23T11:59:56.151204Z",
     "shell.execute_reply.started": "2024-11-23T11:59:56.146078Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : Cleaning data\n",
    "punc_re = re.compile(r\"\"\"[!\"#$%&\\'()*+,-./:;<=>?@[\\\\\\]^_`{|}~،؟…«“\\\":\\\"…”]\"\"\")\n",
    "def remove_punctation(text: str) -> str:\n",
    "    return punc_re.sub(repl=\"\", string=text)\n",
    "\n",
    "\n",
    "diactircs_re = re.compile(\"[\\u064B-\\u0652]\")\n",
    "def remove_diactrics(text: str) -> str:\n",
    "    return diactircs_re.sub(repl=\"\", string=text)\n",
    "\n",
    "def clean_text(text: str) -> str:\n",
    "    return remove_punctation(remove_diactrics(text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:56.362251Z",
     "iopub.status.busy": "2024-11-23T11:59:56.361896Z",
     "iopub.status.idle": "2024-11-23T11:59:56.370514Z",
     "shell.execute_reply": "2024-11-23T11:59:56.369588Z",
     "shell.execute_reply.started": "2024-11-23T11:59:56.362226Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : utilities of creation and preparation data \n",
    "\n",
    "class TranslationDataset(Dataset):\n",
    "    def __init__(self, df: pd.DataFrame):\n",
    "        # Ensure the 'length' column exists\n",
    "        df['length'] = df['source'].apply(len)\n",
    "        self.df = df.sort_values(by=\"length\")\n",
    "        \n",
    "        self.src = self.df[\"source\"].tolist()\n",
    "        self.trg = self.df[\"target\"].tolist()\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        source = self.src[idx]\n",
    "        target = self.trg[idx]\n",
    "        return (source, target)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "\n",
    "def prepare_batch(batch, en_tokenizer: Tokenizer, ur_tokenizer: Tokenizer):\n",
    "    en = list(map(lambda x: x[0], batch))\n",
    "    ur = list(map(lambda x: x[1], batch))\n",
    "    \n",
    "    \n",
    "    en = en_tokenizer.encode_batch(en)\n",
    "    ur = ur_tokenizer.encode_batch(ur)\n",
    "    \n",
    "    \n",
    "    \n",
    "    en_tokens = list(map(lambda x: x.ids, en))\n",
    "    en_mask = list(map(lambda x: x.attention_mask, en))\n",
    "    ur_tokens = list(map(lambda x: x.ids, ur))\n",
    "    ur_mask = list(map(lambda x: x.attention_mask, ur))\n",
    "\n",
    "    \n",
    "    source = torch.tensor(en_tokens)\n",
    "    source_mask = torch.tensor(en_mask)\n",
    "    target = torch.tensor(ur_tokens)\n",
    "    target_mask = torch.tensor(ur_mask)\n",
    "    return (source, source_mask, target, target_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:56.566641Z",
     "iopub.status.busy": "2024-11-23T11:59:56.565750Z",
     "iopub.status.idle": "2024-11-23T11:59:56.570808Z",
     "shell.execute_reply": "2024-11-23T11:59:56.569801Z",
     "shell.execute_reply.started": "2024-11-23T11:59:56.566609Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:56.783992Z",
     "iopub.status.busy": "2024-11-23T11:59:56.783361Z",
     "iopub.status.idle": "2024-11-23T11:59:56.792467Z",
     "shell.execute_reply": "2024-11-23T11:59:56.791717Z",
     "shell.execute_reply.started": "2024-11-23T11:59:56.783964Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        latent_dim: int,\n",
    "        n_heads: int,\n",
    "        dropout: float\n",
    "    ):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.latent_dim = latent_dim\n",
    "        self.n_heads = n_heads\n",
    "        self.head_dim = self.latent_dim // self.n_heads\n",
    "        \n",
    "        self.fc_q = nn.Linear(in_features=self.latent_dim, out_features=self.latent_dim)\n",
    "        self.fc_k = nn.Linear(in_features=self.latent_dim, out_features=self.latent_dim)\n",
    "        self.fc_v = nn.Linear(in_features=self.latent_dim, out_features=self.latent_dim)\n",
    "        \n",
    "        self.fc_o = nn.Linear(in_features=self.latent_dim, out_features=self.latent_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "        \n",
    "    def forward(self, query, key, value, mask = None):\n",
    "        batch_size = query.shape[0]\n",
    "        \n",
    "        Q = self.fc_q(query)\n",
    "        K = self.fc_k(key)\n",
    "        V = self.fc_v(value)\n",
    "        \n",
    "        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        \n",
    "        scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(query)\n",
    "        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / scale\n",
    "        \n",
    "        \n",
    "        if mask is not None:\n",
    "            energy = energy.masked_fill(mask == 0, -1e10)\n",
    "            \n",
    "        attention = self.dropout(torch.softmax(energy, dim=-1))        \n",
    "        x = torch.matmul(attention, V)\n",
    "        x = x.permute(0, 2, 1, 3).contiguous()\n",
    "        x = x.view(batch_size, -1, self.latent_dim)\n",
    "        x = self.fc_o(x)\n",
    "        \n",
    "        return x, attention\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:56.995678Z",
     "iopub.status.busy": "2024-11-23T11:59:56.995018Z",
     "iopub.status.idle": "2024-11-23T11:59:57.000990Z",
     "shell.execute_reply": "2024-11-23T11:59:57.000024Z",
     "shell.execute_reply.started": "2024-11-23T11:59:56.995627Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class PositionWiseFeedForward(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        latent_dim: int,\n",
    "        pf_dim: int,\n",
    "        dropout: float\n",
    "    ):        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(in_features=latent_dim, out_features=pf_dim)\n",
    "        self.fc2 = nn.Linear(in_features=pf_dim, out_features=latent_dim)\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:57.226340Z",
     "iopub.status.busy": "2024-11-23T11:59:57.225950Z",
     "iopub.status.idle": "2024-11-23T11:59:57.234277Z",
     "shell.execute_reply": "2024-11-23T11:59:57.233127Z",
     "shell.execute_reply.started": "2024-11-23T11:59:57.226300Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        latent_dim: int,\n",
    "        n_heads: int,\n",
    "        pf_dim: int,\n",
    "        dropout: float\n",
    "    ):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.attention = MultiHeadAttention(latent_dim=latent_dim, n_heads=n_heads, dropout=dropout)\n",
    "        self.positionwise_feedforward = PositionWiseFeedForward(latent_dim=latent_dim, pf_dim=pf_dim, dropout=dropout)\n",
    "        \n",
    "        self.attn_norm = nn.LayerNorm(normalized_shape=latent_dim)\n",
    "        self.ppff_norm = nn.LayerNorm(normalized_shape=latent_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "    def forward(self, src, src_mask):\n",
    "        attention_values, _ = self.attention(src, src, src, src_mask)        \n",
    "        output = (attention_values + src)        \n",
    "        ppff = self.positionwise_feedforward(output)\n",
    "        output = self.ppff_norm(output + self.dropout(ppff))\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:57.449551Z",
     "iopub.status.busy": "2024-11-23T11:59:57.448904Z",
     "iopub.status.idle": "2024-11-23T11:59:57.456580Z",
     "shell.execute_reply": "2024-11-23T11:59:57.455662Z",
     "shell.execute_reply.started": "2024-11-23T11:59:57.449526Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        vocab_size: int,\n",
    "        latent_dim: int,\n",
    "        n_layers: int,\n",
    "        n_heads: int,\n",
    "        pf_dim: int,\n",
    "        dropout: float\n",
    "    ):\n",
    "        super().__init__()\n",
    "        \n",
    "        \n",
    "        self.tok_embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=latent_dim)\n",
    "        self.pos_embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=latent_dim)\n",
    "        \n",
    "        self.layers = nn.ModuleList(\n",
    "            [\n",
    "                EncoderLayer(\n",
    "                    latent_dim=latent_dim,\n",
    "                    n_heads=n_heads,\n",
    "                    pf_dim=pf_dim,\n",
    "                    dropout=dropout\n",
    "                )\n",
    "                for _ in range(n_layers)]\n",
    "        )\n",
    "    \n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    \n",
    "    def forward(self, src, src_mask):\n",
    "        batch_size, src_len = src.shape\n",
    "        \n",
    "        positions = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(src)\n",
    "        \n",
    "        scale = torch.sqrt(torch.FloatTensor([self.tok_embedding.embedding_dim])).to(src)\n",
    "        token_embeddings = self.tok_embedding(src) * scale\n",
    "        positional_embeddings = self.pos_embedding(positions)\n",
    "        \n",
    "        output = self.dropout(token_embeddings + positional_embeddings)\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            output = layer(output, src_mask)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:57.745680Z",
     "iopub.status.busy": "2024-11-23T11:59:57.745315Z",
     "iopub.status.idle": "2024-11-23T11:59:57.752900Z",
     "shell.execute_reply": "2024-11-23T11:59:57.751968Z",
     "shell.execute_reply.started": "2024-11-23T11:59:57.745633Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        latent_dim: int,\n",
    "        n_heads: int,\n",
    "        pf_dim: int,\n",
    "        dropout: float\n",
    "    ):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.self_attention = MultiHeadAttention(\n",
    "            latent_dim=latent_dim, \n",
    "            n_heads=n_heads,\n",
    "            dropout=dropout\n",
    "        )\n",
    "        self.encoder_attention = MultiHeadAttention(\n",
    "            latent_dim=latent_dim, \n",
    "            n_heads=n_heads, \n",
    "            dropout=dropout\n",
    "        )\n",
    "        self.positionwise_feedfoward = PositionWiseFeedForward(\n",
    "            latent_dim=latent_dim, \n",
    "            pf_dim=pf_dim, \n",
    "            dropout=dropout\n",
    "        )\n",
    "        \n",
    "        self.self_attn_norm = nn.LayerNorm(normalized_shape=latent_dim)\n",
    "        self.enc_attn_norm = nn.LayerNorm(normalized_shape=latent_dim)\n",
    "        self.pff_norm = nn.LayerNorm(normalized_shape=latent_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "    def forward(self, trg, enc_out, trg_mask, src_mask):\n",
    "        self_attention, _ = self.self_attention(trg, trg, trg, trg_mask)\n",
    "        \n",
    "        output = self.self_attn_norm(trg + self.dropout(self_attention))\n",
    "        \n",
    "        encoder_attention, attention_scores = self.encoder_attention(output, enc_out, enc_out, src_mask)\n",
    "        \n",
    "        output = self.enc_attn_norm(output + self.dropout(encoder_attention))\n",
    "        \n",
    "        pwff = self.positionwise_feedfoward(output)\n",
    "        output = self.pff_norm(output + self.dropout(pwff))\n",
    "        \n",
    "        return output, attention_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:57.954335Z",
     "iopub.status.busy": "2024-11-23T11:59:57.954012Z",
     "iopub.status.idle": "2024-11-23T11:59:57.962208Z",
     "shell.execute_reply": "2024-11-23T11:59:57.961265Z",
     "shell.execute_reply.started": "2024-11-23T11:59:57.954311Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        vocab_size: int,\n",
    "        latent_dim: int,\n",
    "        n_layers: int,\n",
    "        n_heads: int,\n",
    "        pf_dim: int,\n",
    "        dropout: float,\n",
    "        max_len: int = 100\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.latent_dim = latent_dim\n",
    "        \n",
    "        self.tok_embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=latent_dim)\n",
    "        self.pos_embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=latent_dim)\n",
    "        \n",
    "        self.layers = nn.ModuleList(\n",
    "            [\n",
    "                DecoderLayer(latent_dim=latent_dim, n_heads=n_heads, pf_dim=pf_dim, dropout=dropout)\n",
    "                for _ in range(n_layers)\n",
    "            ]\n",
    "        )\n",
    "        \n",
    "        self.fc_out = nn.Linear(in_features=latent_dim, out_features=vocab_size)\n",
    "        \n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "    def forward(self, trg, enc_out, trg_mask, src_mask):\n",
    "        \n",
    "        batch_size, trg_len = trg.shape\n",
    "        \n",
    "        positions = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(trg)\n",
    "        \n",
    "        scale = torch.sqrt(torch.FloatTensor([self.latent_dim])).to(trg)\n",
    "        output = (self.tok_embedding(trg) * scale) + self.pos_embedding(positions)\n",
    "        output = self.dropout(output)\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            output, attention = layer(output, enc_out, trg_mask, src_mask)\n",
    "        \n",
    "        output = self.dropout(output)\n",
    "        output = self.fc_out(output)\n",
    "        \n",
    "        return output, attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:58.187117Z",
     "iopub.status.busy": "2024-11-23T11:59:58.186339Z",
     "iopub.status.idle": "2024-11-23T11:59:58.193594Z",
     "shell.execute_reply": "2024-11-23T11:59:58.192715Z",
     "shell.execute_reply.started": "2024-11-23T11:59:58.187087Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        encoder: nn.Module,\n",
    "        decoder: nn.Module,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        \n",
    "    def make_trg_mask(self, trg_pad_mask):\n",
    "        # trg_pad_mask = [batch_size, trg_len]\n",
    "        trg_len = trg_pad_mask.shape[1]\n",
    "        \n",
    "        trg_pad_mask = trg_pad_mask.unsqueeze(1).unsqueeze(2)\n",
    "        \n",
    "        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len))).to(trg_pad_mask).bool()\n",
    "        \n",
    "        trg_mask = trg_pad_mask & trg_sub_mask\n",
    "        \n",
    "        return trg_mask\n",
    "    \n",
    "    def make_src_mask(self, src_pad_mask):\n",
    "        return src_pad_mask.unsqueeze(1).unsqueeze(2)\n",
    "    \n",
    "    def forward(self, src, trg, src_mask, trg_mask):\n",
    "        \n",
    "        src_mask = self.make_src_mask(src_pad_mask=src_mask)\n",
    "        trg_mask = self.make_trg_mask(trg_pad_mask=trg_mask)\n",
    "        encoder_outputs = self.encoder(src, src_mask)\n",
    "        output, attention = self.decoder(trg, encoder_outputs, trg_mask, src_mask)\n",
    "        \n",
    "        return output, attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:58.692460Z",
     "iopub.status.busy": "2024-11-23T11:59:58.691817Z",
     "iopub.status.idle": "2024-11-23T11:59:58.764157Z",
     "shell.execute_reply": "2024-11-23T11:59:58.763297Z",
     "shell.execute_reply.started": "2024-11-23T11:59:58.692431Z"
    },
    "scrolled": true,
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              source  \\\n",
      "0  ﻿The book of the generation of Jesus Christ , ...   \n",
      "1  Abraham begat Isaac ; and Isaac begat Jacob ; ...   \n",
      "2  And Judas begat Phares and Zara of Thamar ; an...   \n",
      "3  And Aram begat Aminadab ; and Aminadab begat N...   \n",
      "4  And Salmon begat Booz of Rachab ; and Booz beg...   \n",
      "\n",
      "                                              target  \n",
      "0         ﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ  \n",
      "1  ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...  \n",
      "2  اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...  \n",
      "3  اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...  \n",
      "4  اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Define file paths\n",
    "file_path_en = \"/kaggle/input/umc005/bible/train.en\"\n",
    "file_path_ur = \"/kaggle/input/umc005/bible/train.ur\"\n",
    "\n",
    "# Read and store the content in DataFrame\n",
    "try:\n",
    "    # Open files\n",
    "    with open(file_path_en, \"r\", encoding=\"utf-8\") as file_en, open(file_path_ur, \"r\", encoding=\"utf-8\") as file_ur:\n",
    "        # Read lines\n",
    "        english_lines = [line.strip() for line in file_en.readlines()]\n",
    "        urdu_lines = [line.strip() for line in file_ur.readlines()]\n",
    "\n",
    "    # Create DataFrame\n",
    "    data = pd.DataFrame({\"source\": english_lines, \"target\": urdu_lines})\n",
    "    \n",
    "    # Display first few rows\n",
    "    print(data.head())\n",
    "except FileNotFoundError as e:\n",
    "    print(f\"File not found: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:59.024294Z",
     "iopub.status.busy": "2024-11-23T11:59:59.023607Z",
     "iopub.status.idle": "2024-11-23T11:59:59.034856Z",
     "shell.execute_reply": "2024-11-23T11:59:59.034065Z",
     "shell.execute_reply.started": "2024-11-23T11:59:59.024267Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>﻿The book of the generation of Jesus Christ , ...</td>\n",
       "      <td>﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abraham begat Isaac ; and Isaac begat Jacob ; ...</td>\n",
       "      <td>ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>And Judas begat Phares and Zara of Thamar ; an...</td>\n",
       "      <td>اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>And Aram begat Aminadab ; and Aminadab begat N...</td>\n",
       "      <td>اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>And Salmon begat Booz of Rachab ; and Booz beg...</td>\n",
       "      <td>اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>And Jesse begat David the king ; and David the...</td>\n",
       "      <td>اور یسی سے داود بادشاہ پیدا ہوا ۔ اور داود سے ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>And Solomon begat Roboam ; and Roboam begat Ab...</td>\n",
       "      <td>اور سلیمان سے رحبعام پیدا ہوا اور رحبعام سے اب...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>And Asa begat Josaphat ; and Josaphat begat Jo...</td>\n",
       "      <td>اور آسا سے یہوسفط پیدا ہوا اور یہوسفط سے یورام...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>And Ozias begat Joatham ; and Joatham begat Ac...</td>\n",
       "      <td>اور عزیاہ سے یوتام پیدا ہوا اور یوتام سے آخز پ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>And Ezekias begat Manasses ; and Manasses bega...</td>\n",
       "      <td>اور حزقیا سے منسی پیدا ہوا اور منسی سے امون پی...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source  \\\n",
       "0  ﻿The book of the generation of Jesus Christ , ...   \n",
       "1  Abraham begat Isaac ; and Isaac begat Jacob ; ...   \n",
       "2  And Judas begat Phares and Zara of Thamar ; an...   \n",
       "3  And Aram begat Aminadab ; and Aminadab begat N...   \n",
       "4  And Salmon begat Booz of Rachab ; and Booz beg...   \n",
       "5  And Jesse begat David the king ; and David the...   \n",
       "6  And Solomon begat Roboam ; and Roboam begat Ab...   \n",
       "7  And Asa begat Josaphat ; and Josaphat begat Jo...   \n",
       "8  And Ozias begat Joatham ; and Joatham begat Ac...   \n",
       "9  And Ezekias begat Manasses ; and Manasses bega...   \n",
       "\n",
       "                                              target  \n",
       "0         ﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ  \n",
       "1  ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...  \n",
       "2  اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...  \n",
       "3  اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...  \n",
       "4  اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...  \n",
       "5  اور یسی سے داود بادشاہ پیدا ہوا ۔ اور داود سے ...  \n",
       "6  اور سلیمان سے رحبعام پیدا ہوا اور رحبعام سے اب...  \n",
       "7  اور آسا سے یہوسفط پیدا ہوا اور یہوسفط سے یورام...  \n",
       "8  اور عزیاہ سے یوتام پیدا ہوا اور یوتام سے آخز پ...  \n",
       "9  اور حزقیا سے منسی پیدا ہوا اور منسی سے امون پی...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TODO : Data Insight\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T11:59:59.938295Z",
     "iopub.status.busy": "2024-11-23T11:59:59.937703Z",
     "iopub.status.idle": "2024-11-23T11:59:59.942315Z",
     "shell.execute_reply": "2024-11-23T11:59:59.941503Z",
     "shell.execute_reply.started": "2024-11-23T11:59:59.938269Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total samples = 7400\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total samples = {len(data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:00.370006Z",
     "iopub.status.busy": "2024-11-23T12:00:00.369628Z",
     "iopub.status.idle": "2024-11-23T12:00:00.439711Z",
     "shell.execute_reply": "2024-11-23T12:00:00.438773Z",
     "shell.execute_reply.started": "2024-11-23T12:00:00.369980Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Duplicated data 16\n",
      "Shape Of Data : (7394, 2)\n"
     ]
    }
   ],
   "source": [
    "# TODO : Apply Preprocessing \n",
    "# 01-Clean\n",
    "data[\"target\"] = data[\"target\"].map(clean_text)\n",
    "data[\"source\"] = data[\"source\"].map(clean_text)\n",
    "\n",
    "# 02-remove duplicate\n",
    "print(f\"Number of Duplicated data {data['target'].duplicated(keep='first').sum()}\")\n",
    "data = data.copy(deep=True)[~data.duplicated()]\n",
    "\n",
    "print(f'Shape Of Data : {data.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:00.644057Z",
     "iopub.status.busy": "2024-11-23T12:00:00.643238Z",
     "iopub.status.idle": "2024-11-23T12:00:00.672551Z",
     "shell.execute_reply": "2024-11-23T12:00:00.671825Z",
     "shell.execute_reply.started": "2024-11-23T12:00:00.644027Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7394.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>26.029619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.497396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>19.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>25.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>32.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>71.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            length\n",
       "count  7394.000000\n",
       "mean     26.029619\n",
       "std       9.497396\n",
       "min       3.000000\n",
       "25%      19.000000\n",
       "50%      25.000000\n",
       "75%      32.000000\n",
       "max      71.000000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TODO : Show Length\n",
    "data[\"length\"] = data[\"source\"].map(lambda x: len(x.split(\" \")))\n",
    "\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:00.808790Z",
     "iopub.status.busy": "2024-11-23T12:00:00.807989Z",
     "iopub.status.idle": "2024-11-23T12:00:00.831639Z",
     "shell.execute_reply": "2024-11-23T12:00:00.830873Z",
     "shell.execute_reply.started": "2024-11-23T12:00:00.808765Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              source  \\\n",
      "0  ﻿The book of the generation of Jesus Christ  t...   \n",
      "1  Abraham begat Isaac  and Isaac begat Jacob  an...   \n",
      "2  And Judas begat Phares and Zara of Thamar  and...   \n",
      "3  And Aram begat Aminadab  and Aminadab begat Na...   \n",
      "4  And Salmon begat Booz of Rachab  and Booz bega...   \n",
      "\n",
      "                                              target  length  \n",
      "0         ﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ      19  \n",
      "1  ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...      17  \n",
      "2  اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...      19  \n",
      "3  اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...      15  \n",
      "4  اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...      19  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Define file paths\n",
    "file_path_en = \"/kaggle/input/umc005/bible/train.en\"\n",
    "file_path_ur = \"/kaggle/input/umc005/bible/train.ur\"\n",
    "\n",
    "# Read and store the content in DataFrame\n",
    "try:\n",
    "    # Open files\n",
    "    with open(file_path_en, \"r\", encoding=\"utf-8\") as file_en, open(file_path_ur, \"r\", encoding=\"utf-8\") as file_ur:\n",
    "        # Read lines\n",
    "        english_lines = [line.strip() for line in file_en.readlines()]\n",
    "        urdu_lines = [line.strip() for line in file_ur.readlines()]\n",
    "\n",
    "    # Create DataFrame\n",
    "    train = pd.DataFrame({\"source\": english_lines, \"target\": urdu_lines})\n",
    "    \n",
    "    # Display first few rows\n",
    "    print(data.head())\n",
    "except FileNotFoundError as e:\n",
    "    print(f\"File not found: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:00.994336Z",
     "iopub.status.busy": "2024-11-23T12:00:00.993762Z",
     "iopub.status.idle": "2024-11-23T12:00:01.002830Z",
     "shell.execute_reply": "2024-11-23T12:00:01.001957Z",
     "shell.execute_reply.started": "2024-11-23T12:00:00.994306Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>﻿The book of the generation of Jesus Christ , ...</td>\n",
       "      <td>﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abraham begat Isaac ; and Isaac begat Jacob ; ...</td>\n",
       "      <td>ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>And Judas begat Phares and Zara of Thamar ; an...</td>\n",
       "      <td>اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>And Aram begat Aminadab ; and Aminadab begat N...</td>\n",
       "      <td>اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>And Salmon begat Booz of Rachab ; and Booz beg...</td>\n",
       "      <td>اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>And Jesse begat David the king ; and David the...</td>\n",
       "      <td>اور یسی سے داود بادشاہ پیدا ہوا ۔ اور داود سے ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>And Solomon begat Roboam ; and Roboam begat Ab...</td>\n",
       "      <td>اور سلیمان سے رحبعام پیدا ہوا اور رحبعام سے اب...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>And Asa begat Josaphat ; and Josaphat begat Jo...</td>\n",
       "      <td>اور آسا سے یہوسفط پیدا ہوا اور یہوسفط سے یورام...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>And Ozias begat Joatham ; and Joatham begat Ac...</td>\n",
       "      <td>اور عزیاہ سے یوتام پیدا ہوا اور یوتام سے آخز پ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>And Ezekias begat Manasses ; and Manasses bega...</td>\n",
       "      <td>اور حزقیا سے منسی پیدا ہوا اور منسی سے امون پی...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source  \\\n",
       "0  ﻿The book of the generation of Jesus Christ , ...   \n",
       "1  Abraham begat Isaac ; and Isaac begat Jacob ; ...   \n",
       "2  And Judas begat Phares and Zara of Thamar ; an...   \n",
       "3  And Aram begat Aminadab ; and Aminadab begat N...   \n",
       "4  And Salmon begat Booz of Rachab ; and Booz beg...   \n",
       "5  And Jesse begat David the king ; and David the...   \n",
       "6  And Solomon begat Roboam ; and Roboam begat Ab...   \n",
       "7  And Asa begat Josaphat ; and Josaphat begat Jo...   \n",
       "8  And Ozias begat Joatham ; and Joatham begat Ac...   \n",
       "9  And Ezekias begat Manasses ; and Manasses bega...   \n",
       "\n",
       "                                              target  \n",
       "0         ﻿یسوع مسیح ابن داود ابن ابرہام کا نسب نامہ  \n",
       "1  ابراہام سے اضحاق پیدا ہوا اور اضحاق سے یعقوب پ...  \n",
       "2  اور یہوداہ سے فارص اور زارح تمر سے پیدا ہوئے ا...  \n",
       "3  اور رام سے عمینداب پیدا ہوا اور عمینداب سے نحس...  \n",
       "4  اور سلمون سے بوعز راحب سے پیدا ہوا اور بوعز سے...  \n",
       "5  اور یسی سے داود بادشاہ پیدا ہوا ۔ اور داود سے ...  \n",
       "6  اور سلیمان سے رحبعام پیدا ہوا اور رحبعام سے اب...  \n",
       "7  اور آسا سے یہوسفط پیدا ہوا اور یہوسفط سے یورام...  \n",
       "8  اور عزیاہ سے یوتام پیدا ہوا اور یوتام سے آخز پ...  \n",
       "9  اور حزقیا سے منسی پیدا ہوا اور منسی سے امون پی...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:01.219439Z",
     "iopub.status.busy": "2024-11-23T12:00:01.218585Z",
     "iopub.status.idle": "2024-11-23T12:00:01.243900Z",
     "shell.execute_reply": "2024-11-23T12:00:01.243046Z",
     "shell.execute_reply.started": "2024-11-23T12:00:01.219412Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              source  \\\n",
      "0  And to them it was given that they should not ...   \n",
      "1  And in those days shall men seek death , and s...   \n",
      "2  And the shapes of the locusts were like unto h...   \n",
      "3  And they had hair as the hair of women , and t...   \n",
      "4  And they had breastplates , as it were breastp...   \n",
      "\n",
      "                                              target  \n",
      "0  اور انہیں جان سے مارنے کا نہیں بلکہ پانچ مہینے...  \n",
      "1  ان دنوں میں آدمی موت ڈھونڈیں گے مگر ہرگز نہ پا...  \n",
      "2  اور ان ڈڈیوں کی صورتیں ان گھوڑوں کی سی تھیں جو...  \n",
      "3      اور بال عورتوں کے سے تھے اور دانت ببر کے سے ۔  \n",
      "4  ان کے پاس لوہے کے سے بکتر تھے اور ان کے پروں ک...  \n"
     ]
    }
   ],
   "source": [
    "# # TODO : Shuffle and Split Data\n",
    "# train, test = train_test_split(data,random_state=42)\n",
    "# len(train), len(test)\n",
    "import pandas as pd\n",
    "\n",
    "# Define file paths for test data\n",
    "file_path_test_en = \"/kaggle/input/umc005/bible/test.en\"\n",
    "file_path_test_ur = \"/kaggle/input/umc005/bible/test.ur\"\n",
    "\n",
    "# Read and store the content in DataFrame\n",
    "try:\n",
    "    # Open test files\n",
    "    with open(file_path_test_en, \"r\", encoding=\"utf-8\") as file_test_en, open(file_path_test_ur, \"r\", encoding=\"utf-8\") as file_test_ur:\n",
    "        # Read lines from test files\n",
    "        test_english_lines = [line.strip() for line in file_test_en.readlines()]\n",
    "        test_urdu_lines = [line.strip() for line in file_test_ur.readlines()]\n",
    "\n",
    "    # Create test DataFrame\n",
    "    test = pd.DataFrame({\"source\": test_english_lines, \"target\": test_urdu_lines})\n",
    "    \n",
    "    # Display first few rows of the test DataFrame\n",
    "    print(test.head())\n",
    "except FileNotFoundError as e:\n",
    "    print(f\"File not found: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:01.382242Z",
     "iopub.status.busy": "2024-11-23T12:00:01.381924Z",
     "iopub.status.idle": "2024-11-23T12:00:01.390611Z",
     "shell.execute_reply": "2024-11-23T12:00:01.389640Z",
     "shell.execute_reply.started": "2024-11-23T12:00:01.382218Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>And to them it was given that they should not ...</td>\n",
       "      <td>اور انہیں جان سے مارنے کا نہیں بلکہ پانچ مہینے...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>And in those days shall men seek death , and s...</td>\n",
       "      <td>ان دنوں میں آدمی موت ڈھونڈیں گے مگر ہرگز نہ پا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>And the shapes of the locusts were like unto h...</td>\n",
       "      <td>اور ان ڈڈیوں کی صورتیں ان گھوڑوں کی سی تھیں جو...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>And they had hair as the hair of women , and t...</td>\n",
       "      <td>اور بال عورتوں کے سے تھے اور دانت ببر کے سے ۔</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>And they had breastplates , as it were breastp...</td>\n",
       "      <td>ان کے پاس لوہے کے سے بکتر تھے اور ان کے پروں ک...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>And they had tails like unto scorpions , and t...</td>\n",
       "      <td>اور ان کی دمیں بچھوؤں کی سی تھیں اور ان میں ڈ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>And they had a king over them , which is the a...</td>\n",
       "      <td>اتھاہ گڑھے کا فرشتہ ان پر بادشاہ تھا ۔ اس کا ن...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>One woe is past ; and , behold , there come tw...</td>\n",
       "      <td>پہلا افسوس تو ہو چکا ۔ دیکھو اس کے بعد دو افسو...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>And the sixth angel sounded , and I heard a vo...</td>\n",
       "      <td>اور جب چھٹے فرشتہ نے نرسنگا پھونکا تو میں نے ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Saying to the sixth angel which had the trumpe...</td>\n",
       "      <td>کہ اس چھٹے فرشتہ سے جس کے پاس نرسنگا تھا کوئی ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source  \\\n",
       "0  And to them it was given that they should not ...   \n",
       "1  And in those days shall men seek death , and s...   \n",
       "2  And the shapes of the locusts were like unto h...   \n",
       "3  And they had hair as the hair of women , and t...   \n",
       "4  And they had breastplates , as it were breastp...   \n",
       "5  And they had tails like unto scorpions , and t...   \n",
       "6  And they had a king over them , which is the a...   \n",
       "7  One woe is past ; and , behold , there come tw...   \n",
       "8  And the sixth angel sounded , and I heard a vo...   \n",
       "9  Saying to the sixth angel which had the trumpe...   \n",
       "\n",
       "                                              target  \n",
       "0  اور انہیں جان سے مارنے کا نہیں بلکہ پانچ مہینے...  \n",
       "1  ان دنوں میں آدمی موت ڈھونڈیں گے مگر ہرگز نہ پا...  \n",
       "2  اور ان ڈڈیوں کی صورتیں ان گھوڑوں کی سی تھیں جو...  \n",
       "3      اور بال عورتوں کے سے تھے اور دانت ببر کے سے ۔  \n",
       "4  ان کے پاس لوہے کے سے بکتر تھے اور ان کے پروں ک...  \n",
       "5  اور ان کی دمیں بچھوؤں کی سی تھیں اور ان میں ڈ...  \n",
       "6  اتھاہ گڑھے کا فرشتہ ان پر بادشاہ تھا ۔ اس کا ن...  \n",
       "7  پہلا افسوس تو ہو چکا ۔ دیکھو اس کے بعد دو افسو...  \n",
       "8  اور جب چھٹے فرشتہ نے نرسنگا پھونکا تو میں نے ا...  \n",
       "9  کہ اس چھٹے فرشتہ سے جس کے پاس نرسنگا تھا کوئی ...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:01.804713Z",
     "iopub.status.busy": "2024-11-23T12:00:01.804350Z",
     "iopub.status.idle": "2024-11-23T12:00:01.810559Z",
     "shell.execute_reply": "2024-11-23T12:00:01.809727Z",
     "shell.execute_reply.started": "2024-11-23T12:00:01.804680Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : tokenization\n",
    "normalizer = normalizers.Sequence([NFC(), StripAccents(), Lowercase(), Strip()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:02.011233Z",
     "iopub.status.busy": "2024-11-23T12:00:02.010903Z",
     "iopub.status.idle": "2024-11-23T12:00:02.024993Z",
     "shell.execute_reply": "2024-11-23T12:00:02.024118Z",
     "shell.execute_reply.started": "2024-11-23T12:00:02.011209Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : add spcial tokens\n",
    "post_processor = TemplateProcessing(\n",
    "    single=\"[SOS] $A [EOS]\",\n",
    "    special_tokens=[(\"[SOS]\", 2), (\"[EOS]\", 3)]\n",
    ")\n",
    "\n",
    "en_tokenizer = Tokenizer(model=WordLevel(unk_token=\"[OOV]\"))\n",
    "ur_tokenizer = Tokenizer(model=WordLevel(unk_token=\"[OOV]\"))\n",
    "\n",
    "pre_tokenizer = pre_tokenizers.Sequence([Whitespace()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:02.190046Z",
     "iopub.status.busy": "2024-11-23T12:00:02.189716Z",
     "iopub.status.idle": "2024-11-23T12:00:02.195054Z",
     "shell.execute_reply": "2024-11-23T12:00:02.193957Z",
     "shell.execute_reply.started": "2024-11-23T12:00:02.190021Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "en_tokenizer.normalizer = normalizer\n",
    "en_tokenizer.pre_tokenizer = pre_tokenizer\n",
    "\n",
    "ur_tokenizer.normalizer = normalizer\n",
    "ur_tokenizer.pre_tokenizer = pre_tokenizer\n",
    "\n",
    "en_tokenizer.enable_padding(direction='right')\n",
    "ur_tokenizer.enable_padding(direction='right')\n",
    "\n",
    "\n",
    "en_tokenizer.post_processor = post_processor\n",
    "ur_tokenizer.post_processor = post_processor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:02.390114Z",
     "iopub.status.busy": "2024-11-23T12:00:02.389778Z",
     "iopub.status.idle": "2024-11-23T12:00:02.647435Z",
     "shell.execute_reply": "2024-11-23T12:00:02.646696Z",
     "shell.execute_reply.started": "2024-11-23T12:00:02.390090Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "trainer = WordLevelTrainer(vocab_size=200000, show_progress=False, min_frequency=5, \n",
    "                           special_tokens=[\"[PAD]\", \"[OOV]\", \"[SOS]\", \"[EOS]\"]\n",
    "                          )\n",
    "\n",
    "en_tokenizer.train_from_iterator(iterator=data[\"source\"].tolist(), trainer=trainer)\n",
    "ur_tokenizer.train_from_iterator(iterator=data[\"target\"].tolist(), trainer=trainer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:02.648943Z",
     "iopub.status.busy": "2024-11-23T12:00:02.648534Z",
     "iopub.status.idle": "2024-11-23T12:00:02.657600Z",
     "shell.execute_reply": "2024-11-23T12:00:02.656802Z",
     "shell.execute_reply.started": "2024-11-23T12:00:02.648918Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('nevertheless', 523),\n",
       " ('nay', 720),\n",
       " ('escaped', 1215),\n",
       " ('bowed', 1831),\n",
       " ('parted', 1549),\n",
       " ('pharisee', 1012),\n",
       " ('earnest', 1396),\n",
       " ('covetous', 1389),\n",
       " ('coming', 306),\n",
       " ('idle', 1526)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " list(en_tokenizer.get_vocab().items())[:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:02.827815Z",
     "iopub.status.busy": "2024-11-23T12:00:02.827009Z",
     "iopub.status.idle": "2024-11-23T12:00:02.834356Z",
     "shell.execute_reply": "2024-11-23T12:00:02.833399Z",
     "shell.execute_reply.started": "2024-11-23T12:00:02.827790Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('لائیں', 905),\n",
       " ('کہ', 14),\n",
       " ('روٹی', 369),\n",
       " ('قوموں', 402),\n",
       " ('بھوکا', 1224),\n",
       " ('یہاں', 230),\n",
       " ('زریعہ', 1758),\n",
       " ('اٹھکر', 2068),\n",
       " ('روانہ', 342),\n",
       " ('پچھلا', 1659)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " list(ur_tokenizer.get_vocab().items())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:03.037772Z",
     "iopub.status.busy": "2024-11-23T12:00:03.037423Z",
     "iopub.status.idle": "2024-11-23T12:00:03.043152Z",
     "shell.execute_reply": "2024-11-23T12:00:03.042313Z",
     "shell.execute_reply.started": "2024-11-23T12:00:03.037745Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ar vocab size : 2078 \n",
      "En vocab size 2310 \n"
     ]
    }
   ],
   "source": [
    "print(f'Ar vocab size : {en_tokenizer.get_vocab_size()} \\nEn vocab size {ur_tokenizer.get_vocab_size()} ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:03.237855Z",
     "iopub.status.busy": "2024-11-23T12:00:03.237504Z",
     "iopub.status.idle": "2024-11-23T12:00:03.251624Z",
     "shell.execute_reply": "2024-11-23T12:00:03.250911Z",
     "shell.execute_reply.started": "2024-11-23T12:00:03.237829Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : Apply DataSet \n",
    "train_dataset = TranslationDataset(df=train)\n",
    "test_dataset = TranslationDataset(df=test)\n",
    "\n",
    "\n",
    "train_dataloader = DataLoader(dataset=train_dataset, batch_size=var.TEXT['batch'],\n",
    "                              collate_fn=lambda x: prepare_batch(x, en_tokenizer, ur_tokenizer)) \n",
    "    \n",
    "test_dataloader = DataLoader(dataset=test_dataset, batch_size=var.TEXT['batch'], \n",
    "                             collate_fn=lambda x: prepare_batch(x, en_tokenizer, ur_tokenizer)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:03.477237Z",
     "iopub.status.busy": "2024-11-23T12:00:03.476910Z",
     "iopub.status.idle": "2024-11-23T12:00:03.482930Z",
     "shell.execute_reply": "2024-11-23T12:00:03.482052Z",
     "shell.execute_reply.started": "2024-11-23T12:00:03.477215Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(232, 9)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TODO : Datasets Insight\n",
    "len(train_dataloader) , len(test_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:03.678197Z",
     "iopub.status.busy": "2024-11-23T12:00:03.677393Z",
     "iopub.status.idle": "2024-11-23T12:00:04.120583Z",
     "shell.execute_reply": "2024-11-23T12:00:04.119874Z",
     "shell.execute_reply.started": "2024-11-23T12:00:03.678170Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : Model instance \n",
    "\n",
    "\n",
    "encoder = Encoder(\n",
    "    vocab_size = en_tokenizer.get_vocab_size(), \n",
    "    latent_dim = var.TRANSFORMER['latent_dim'],\n",
    "    n_layers   = var.TRANSFORMER['encoder_layers'],\n",
    "    n_heads    = var.TRANSFORMER['heads'],\n",
    "    pf_dim     = var.TRANSFORMER['pf_dim'],\n",
    "    dropout    = var.TRANSFORMER['dropout']\n",
    ")\n",
    "\n",
    "decoder = Decoder(\n",
    "    vocab_size = ur_tokenizer.get_vocab_size(),\n",
    "    latent_dim = var.TRANSFORMER['latent_dim'],\n",
    "    n_layers   = var.TRANSFORMER['decoder_layers'], \n",
    "    n_heads    = var.TRANSFORMER['heads'],\n",
    "    pf_dim     = var.TRANSFORMER['pf_dim'],\n",
    "    dropout    = var.TRANSFORMER['dropout']\n",
    ")\n",
    "\n",
    "model = Seq2Seq(encoder=encoder, decoder=decoder).to(var.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:04.122132Z",
     "iopub.status.busy": "2024-11-23T12:00:04.121873Z",
     "iopub.status.idle": "2024-11-23T12:00:04.126886Z",
     "shell.execute_reply": "2024-11-23T12:00:04.125987Z",
     "shell.execute_reply.started": "2024-11-23T12:00:04.122111Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of trainable parameter : 21449990\n"
     ]
    }
   ],
   "source": [
    "# TODO : Model Summary and plot\n",
    "print(f'number of trainable parameter : {count_parameters(model)}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:04.369431Z",
     "iopub.status.busy": "2024-11-23T12:00:04.369096Z",
     "iopub.status.idle": "2024-11-23T12:00:04.385235Z",
     "shell.execute_reply": "2024-11-23T12:00:04.384157Z",
     "shell.execute_reply.started": "2024-11-23T12:00:04.369406Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# init_weights\n",
    "def initialize_weights(m):\n",
    "    if hasattr(m, 'weight') and m.weight.dim() > 1:\n",
    "        nn.init.xavier_uniform_(m.weight.data)\n",
    "        \n",
    "model.apply(initialize_weights);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:04.621387Z",
     "iopub.status.busy": "2024-11-23T12:00:04.621060Z",
     "iopub.status.idle": "2024-11-23T12:00:04.626869Z",
     "shell.execute_reply": "2024-11-23T12:00:04.625990Z",
     "shell.execute_reply.started": "2024-11-23T12:00:04.621361Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# TODO : Optimizer  \n",
    "optim = torch.optim.Adam(params=model.parameters(), lr=var.TRAIN['lr'])\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T12:00:04.843468Z",
     "iopub.status.busy": "2024-11-23T12:00:04.842890Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 1/500: 100%|██████████| 232/232 [00:10<00:00, 22.48batch/s, train_loss=5.71]\n",
      "Validation Epoch 1/500: 100%|██████████| 9/9 [00:00<00:00, 48.43batch/s, val_loss=6.4] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1\tTrain Loss = 5.6860\tValidation Loss = 4.9756\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 2/500: 100%|██████████| 232/232 [00:09<00:00, 24.46batch/s, train_loss=4.91]\n",
      "Validation Epoch 2/500: 100%|██████████| 9/9 [00:00<00:00, 52.71batch/s, val_loss=5.95]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2\tTrain Loss = 4.9074\tValidation Loss = 4.6266\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 3/500: 100%|██████████| 232/232 [00:09<00:00, 24.45batch/s, train_loss=4.52]\n",
      "Validation Epoch 3/500: 100%|██████████| 9/9 [00:00<00:00, 53.54batch/s, val_loss=5.74]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3\tTrain Loss = 4.5209\tValidation Loss = 4.4623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 4/500: 100%|██████████| 232/232 [00:09<00:00, 24.20batch/s, train_loss=4.3] \n",
      "Validation Epoch 4/500: 100%|██████████| 9/9 [00:00<00:00, 52.96batch/s, val_loss=5.62]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4\tTrain Loss = 4.2857\tValidation Loss = 4.3690\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 15/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=3.08]\n",
      "Validation Epoch 15/500: 100%|██████████| 9/9 [00:00<00:00, 54.90batch/s, val_loss=5.12]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15\tTrain Loss = 3.0683\tValidation Loss = 3.9794\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 16/500: 100%|██████████| 232/232 [00:09<00:00, 24.60batch/s, train_loss=2.99]\n",
      "Validation Epoch 16/500: 100%|██████████| 9/9 [00:00<00:00, 54.29batch/s, val_loss=5.11]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16\tTrain Loss = 2.9931\tValidation Loss = 3.9776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 17/500: 100%|██████████| 232/232 [00:09<00:00, 24.29batch/s, train_loss=2.93]\n",
      "Validation Epoch 17/500: 100%|██████████| 9/9 [00:00<00:00, 53.74batch/s, val_loss=5.1] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17\tTrain Loss = 2.9269\tValidation Loss = 3.9633\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 18/500: 100%|██████████| 232/232 [00:09<00:00, 24.61batch/s, train_loss=2.85]\n",
      "Validation Epoch 18/500: 100%|██████████| 9/9 [00:00<00:00, 55.24batch/s, val_loss=5.09]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18\tTrain Loss = 2.8543\tValidation Loss = 3.9619\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 19/500: 100%|██████████| 232/232 [00:09<00:00, 24.67batch/s, train_loss=2.8] \n",
      "Validation Epoch 19/500: 100%|██████████| 9/9 [00:00<00:00, 54.86batch/s, val_loss=5.13]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19\tTrain Loss = 2.7869\tValidation Loss = 3.9938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 20/500: 100%|██████████| 232/232 [00:09<00:00, 24.44batch/s, train_loss=2.72]\n",
      "Validation Epoch 20/500: 100%|██████████| 9/9 [00:00<00:00, 54.33batch/s, val_loss=5.15]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20\tTrain Loss = 2.7233\tValidation Loss = 4.0073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 21/500: 100%|██████████| 232/232 [00:09<00:00, 24.64batch/s, train_loss=2.66]\n",
      "Validation Epoch 21/500: 100%|██████████| 9/9 [00:00<00:00, 53.16batch/s, val_loss=5.17]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21\tTrain Loss = 2.6604\tValidation Loss = 4.0202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 22/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=2.6] \n",
      "Validation Epoch 22/500: 100%|██████████| 9/9 [00:00<00:00, 53.53batch/s, val_loss=5.14]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22\tTrain Loss = 2.6038\tValidation Loss = 4.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 23/500: 100%|██████████| 232/232 [00:09<00:00, 24.44batch/s, train_loss=2.55]\n",
      "Validation Epoch 23/500: 100%|██████████| 9/9 [00:00<00:00, 45.76batch/s, val_loss=5.18]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23\tTrain Loss = 2.5431\tValidation Loss = 4.0278\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 24/500:  73%|███████▎  | 170/232 [00:06<00:02, 23.00batch/s, train_loss=2.37]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 25/500: 100%|██████████| 232/232 [00:09<00:00, 24.28batch/s, train_loss=2.43]\n",
      "Training Epoch 34/500: 100%|██████████| 232/232 [00:09<00:00, 24.41batch/s, train_loss=1.99]\n",
      "Validation Epoch 34/500: 100%|██████████| 9/9 [00:00<00:00, 53.91batch/s, val_loss=5.48]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34\tTrain Loss = 1.9933\tValidation Loss = 4.2633\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 35/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=1.95]\n",
      "Validation Epoch 35/500: 100%|██████████| 9/9 [00:00<00:00, 53.64batch/s, val_loss=5.49]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35\tTrain Loss = 1.9476\tValidation Loss = 4.2666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 36/500: 100%|██████████| 232/232 [00:09<00:00, 24.62batch/s, train_loss=1.91]\n",
      "Validation Epoch 36/500: 100%|██████████| 9/9 [00:00<00:00, 46.01batch/s, val_loss=6.47]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36\tTrain Loss = 1.9062\tValidation Loss = 4.3110\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 37/500: 100%|██████████| 232/232 [00:09<00:00, 24.34batch/s, train_loss=1.88]\n",
      "Validation Epoch 37/500: 100%|██████████| 9/9 [00:00<00:00, 54.09batch/s, val_loss=5.59]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37\tTrain Loss = 1.8673\tValidation Loss = 4.3496\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 38/500: 100%|██████████| 232/232 [00:09<00:00, 24.54batch/s, train_loss=1.83]\n",
      "Validation Epoch 38/500: 100%|██████████| 9/9 [00:00<00:00, 54.54batch/s, val_loss=5.63]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38\tTrain Loss = 1.8301\tValidation Loss = 4.3807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 39/500: 100%|██████████| 232/232 [00:09<00:00, 24.64batch/s, train_loss=1.79]\n",
      "Validation Epoch 39/500: 100%|██████████| 9/9 [00:00<00:00, 54.58batch/s, val_loss=5.66]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39\tTrain Loss = 1.7857\tValidation Loss = 4.4052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 40/500: 100%|██████████| 232/232 [00:09<00:00, 24.42batch/s, train_loss=1.75]\n",
      "Validation Epoch 40/500: 100%|██████████| 9/9 [00:00<00:00, 54.04batch/s, val_loss=5.73]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40\tTrain Loss = 1.7459\tValidation Loss = 4.4529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 41/500: 100%|██████████| 232/232 [00:09<00:00, 24.58batch/s, train_loss=1.72]\n",
      "Validation Epoch 41/500: 100%|██████████| 9/9 [00:00<00:00, 52.96batch/s, val_loss=5.75]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41\tTrain Loss = 1.7130\tValidation Loss = 4.4721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 42/500: 100%|██████████| 232/232 [00:09<00:00, 24.58batch/s, train_loss=1.68]\n",
      "Validation Epoch 42/500: 100%|██████████| 9/9 [00:00<00:00, 54.04batch/s, val_loss=5.78]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42\tTrain Loss = 1.6743\tValidation Loss = 4.4977\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 43/500: 100%|██████████| 232/232 [00:09<00:00, 24.38batch/s, train_loss=1.64]\n",
      "Validation Epoch 43/500: 100%|██████████| 9/9 [00:00<00:00, 55.27batch/s, val_loss=5.85]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43\tTrain Loss = 1.6337\tValidation Loss = 4.5465\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 44/500:  17%|█▋        | 40/232 [00:01<00:06, 30.79batch/s, train_loss=1.04] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 61/500: 100%|██████████| 232/232 [00:09<00:00, 24.68batch/s, train_loss=1.13] \n",
      "Validation Epoch 61/500: 100%|██████████| 9/9 [00:00<00:00, 54.83batch/s, val_loss=6.63]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61\tTrain Loss = 1.1221\tValidation Loss = 5.1556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 62/500: 100%|██████████| 232/232 [00:09<00:00, 24.55batch/s, train_loss=1.11] \n",
      "Validation Epoch 62/500: 100%|██████████| 9/9 [00:00<00:00, 54.21batch/s, val_loss=6.65]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62\tTrain Loss = 1.1006\tValidation Loss = 5.1756\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 84/500: 100%|██████████| 232/232 [00:09<00:00, 24.52batch/s, train_loss=0.726]\n",
      "Validation Epoch 84/500: 100%|██████████| 9/9 [00:00<00:00, 52.95batch/s, val_loss=7.44]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84\tTrain Loss = 0.7258\tValidation Loss = 5.7900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 85/500: 100%|██████████| 232/232 [00:09<00:00, 24.49batch/s, train_loss=0.706]\n",
      "Validation Epoch 85/500: 100%|██████████| 9/9 [00:00<00:00, 53.77batch/s, val_loss=7.48]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85\tTrain Loss = 0.7058\tValidation Loss = 5.8197\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 86/500: 100%|██████████| 232/232 [00:09<00:00, 24.34batch/s, train_loss=0.7]  \n",
      "Validation Epoch 86/500: 100%|██████████| 9/9 [00:00<00:00, 54.19batch/s, val_loss=7.57]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86\tTrain Loss = 0.7005\tValidation Loss = 5.8880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 87/500: 100%|██████████| 232/232 [00:09<00:00, 24.59batch/s, train_loss=0.687]\n",
      "Validation Epoch 87/500: 100%|██████████| 9/9 [00:00<00:00, 53.82batch/s, val_loss=7.61]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87\tTrain Loss = 0.6840\tValidation Loss = 5.9220\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 88/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.674]\n",
      "Validation Epoch 88/500: 100%|██████████| 9/9 [00:00<00:00, 53.35batch/s, val_loss=7.65]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88\tTrain Loss = 0.6710\tValidation Loss = 5.9462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 89/500: 100%|██████████| 232/232 [00:09<00:00, 24.31batch/s, train_loss=0.659]\n",
      "Validation Epoch 89/500: 100%|██████████| 9/9 [00:00<00:00, 53.82batch/s, val_loss=7.65]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89\tTrain Loss = 0.6595\tValidation Loss = 5.9492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 90/500: 100%|██████████| 232/232 [00:09<00:00, 24.65batch/s, train_loss=0.652]\n",
      "Validation Epoch 90/500: 100%|██████████| 9/9 [00:00<00:00, 54.69batch/s, val_loss=7.77]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90\tTrain Loss = 0.6489\tValidation Loss = 6.0440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 91/500: 100%|██████████| 232/232 [00:09<00:00, 24.53batch/s, train_loss=0.635]\n",
      "Validation Epoch 91/500: 100%|██████████| 9/9 [00:00<00:00, 54.04batch/s, val_loss=7.73]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91\tTrain Loss = 0.6346\tValidation Loss = 6.0088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 92/500: 100%|██████████| 232/232 [00:09<00:00, 24.33batch/s, train_loss=0.63] \n",
      "Validation Epoch 92/500: 100%|██████████| 9/9 [00:00<00:00, 53.99batch/s, val_loss=7.81]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92\tTrain Loss = 0.6273\tValidation Loss = 6.0744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 93/500: 100%|██████████| 232/232 [00:09<00:00, 24.62batch/s, train_loss=0.621]\n",
      "Validation Epoch 93/500:  67%|██████▋   | 6/9 [00:00<00:00, 55.14batch/s, val_loss=35.3]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 112/500: 100%|██████████| 232/232 [00:09<00:00, 24.47batch/s, train_loss=0.455]\n",
      "Validation Epoch 112/500: 100%|██████████| 9/9 [00:00<00:00, 54.85batch/s, val_loss=8.51]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 112\tTrain Loss = 0.4547\tValidation Loss = 6.6213\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 113/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.456]\n",
      "Training Epoch 137/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.337]\n",
      "Validation Epoch 137/500: 100%|██████████| 9/9 [00:00<00:00, 52.42batch/s, val_loss=9.38]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 137\tTrain Loss = 0.3366\tValidation Loss = 7.2992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 138/500: 100%|██████████| 232/232 [00:09<00:00, 24.41batch/s, train_loss=0.341]\n",
      "Validation Epoch 138/500: 100%|██████████| 9/9 [00:00<00:00, 47.96batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 138\tTrain Loss = 0.3394\tValidation Loss = 7.2768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 139/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.331]\n",
      "Validation Epoch 139/500: 100%|██████████| 9/9 [00:00<00:00, 53.82batch/s, val_loss=9.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 139\tTrain Loss = 0.3294\tValidation Loss = 7.2814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 140/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.324]\n",
      "Validation Epoch 140/500: 100%|██████████| 9/9 [00:00<00:00, 54.89batch/s, val_loss=9.31]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 140\tTrain Loss = 0.3229\tValidation Loss = 7.2403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 141/500: 100%|██████████| 232/232 [00:09<00:00, 24.48batch/s, train_loss=0.329]\n",
      "Validation Epoch 141/500: 100%|██████████| 9/9 [00:00<00:00, 54.84batch/s, val_loss=9.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 141\tTrain Loss = 0.3272\tValidation Loss = 7.2885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 142/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=0.32] \n",
      "Validation Epoch 142/500: 100%|██████████| 9/9 [00:00<00:00, 55.26batch/s, val_loss=9.42]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 142\tTrain Loss = 0.3203\tValidation Loss = 7.3303\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 143/500: 100%|██████████| 232/232 [00:09<00:00, 24.67batch/s, train_loss=0.318]\n",
      "Validation Epoch 143/500: 100%|██████████| 9/9 [00:00<00:00, 53.88batch/s, val_loss=9.35]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 143\tTrain Loss = 0.3168\tValidation Loss = 7.2706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 144/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.314]\n",
      "Validation Epoch 144/500: 100%|██████████| 9/9 [00:00<00:00, 54.49batch/s, val_loss=9.4] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 144\tTrain Loss = 0.3145\tValidation Loss = 7.3141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 145/500: 100%|██████████| 232/232 [00:09<00:00, 24.27batch/s, train_loss=0.313]\n",
      "Validation Epoch 145/500: 100%|██████████| 9/9 [00:00<00:00, 54.38batch/s, val_loss=9.42]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 145\tTrain Loss = 0.3119\tValidation Loss = 7.3286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 146/500:  36%|███▌      | 83/232 [00:02<00:05, 26.43batch/s, train_loss=0.215]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 165/500: 100%|██████████| 232/232 [00:09<00:00, 24.75batch/s, train_loss=0.256]\n",
      "Validation Epoch 165/500: 100%|██████████| 9/9 [00:00<00:00, 55.49batch/s, val_loss=9.81]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 165\tTrain Loss = 0.2557\tValidation Loss = 7.6308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 166/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.257]\n",
      "Validation Epoch 166/500: 100%|██████████| 9/9 [00:00<00:00, 54.66batch/s, val_loss=9.78]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 166\tTrain Loss = 0.2558\tValidation Loss = 7.6080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 167/500: 100%|██████████| 232/232 [00:09<00:00, 24.73batch/s, train_loss=0.254]\n",
      "Validation Epoch 167/500: 100%|██████████| 9/9 [00:00<00:00, 54.79batch/s, val_loss=9.87]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 167\tTrain Loss = 0.2532\tValidation Loss = 7.6765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 192/500: 100%|██████████| 232/232 [00:09<00:00, 24.64batch/s, train_loss=0.208]\n",
      "Validation Epoch 192/500: 100%|██████████| 9/9 [00:00<00:00, 53.91batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 192\tTrain Loss = 0.2077\tValidation Loss = 7.9221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 193/500: 100%|██████████| 232/232 [00:09<00:00, 24.50batch/s, train_loss=0.208]\n",
      "Validation Epoch 193/500: 100%|██████████| 9/9 [00:00<00:00, 54.61batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 193\tTrain Loss = 0.2084\tValidation Loss = 7.9036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 194/500: 100%|██████████| 232/232 [00:09<00:00, 24.37batch/s, train_loss=0.207]\n",
      "Validation Epoch 194/500: 100%|██████████| 9/9 [00:00<00:00, 54.42batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 194\tTrain Loss = 0.2067\tValidation Loss = 7.9373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 195/500: 100%|██████████| 232/232 [00:09<00:00, 24.60batch/s, train_loss=0.209]\n",
      "Validation Epoch 195/500: 100%|██████████| 9/9 [00:00<00:00, 54.65batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 195\tTrain Loss = 0.2078\tValidation Loss = 7.9103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 196/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.203]\n",
      "Validation Epoch 196/500: 100%|██████████| 9/9 [00:00<00:00, 54.68batch/s, val_loss=10.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 196\tTrain Loss = 0.2035\tValidation Loss = 7.8581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 197/500: 100%|██████████| 232/232 [00:09<00:00, 24.45batch/s, train_loss=0.203]\n",
      "Validation Epoch 197/500: 100%|██████████| 9/9 [00:00<00:00, 54.46batch/s, val_loss=10.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 197\tTrain Loss = 0.2021\tValidation Loss = 7.8529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 198/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.197]\n",
      "Validation Epoch 198/500: 100%|██████████| 9/9 [00:00<00:00, 55.10batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 198\tTrain Loss = 0.1965\tValidation Loss = 7.9481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 199/500: 100%|██████████| 232/232 [00:09<00:00, 24.74batch/s, train_loss=0.201]\n",
      "Validation Epoch 199/500: 100%|██████████| 9/9 [00:00<00:00, 55.67batch/s, val_loss=10.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 199\tTrain Loss = 0.2011\tValidation Loss = 7.8681\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 200/500: 100%|██████████| 232/232 [00:09<00:00, 24.70batch/s, train_loss=0.2]  \n",
      "Validation Epoch 200/500: 100%|██████████| 9/9 [00:00<00:00, 52.05batch/s, val_loss=10.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 200\tTrain Loss = 0.2001\tValidation Loss = 7.8760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 201/500: 100%|██████████| 232/232 [00:09<00:00, 24.22batch/s, train_loss=0.197]\n",
      "Validation Epoch 201/500: 100%|██████████| 9/9 [00:00<00:00, 54.72batch/s, val_loss=10.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 201\tTrain Loss = 0.1975\tValidation Loss = 7.8959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 202/500:  10%|█         | 24/232 [00:00<00:06, 31.95batch/s, train_loss=0.12] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 217/500: 100%|██████████| 232/232 [00:09<00:00, 24.41batch/s, train_loss=0.179]\n",
      "Validation Epoch 217/500: 100%|██████████| 9/9 [00:00<00:00, 54.31batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 217\tTrain Loss = 0.1778\tValidation Loss = 8.1409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 218/500: 100%|██████████| 232/232 [00:09<00:00, 24.65batch/s, train_loss=0.179]\n",
      "Validation Epoch 218/500: 100%|██████████| 9/9 [00:00<00:00, 54.49batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 218\tTrain Loss = 0.1792\tValidation Loss = 8.1867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 219/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=0.18] \n",
      "Validation Epoch 219/500: 100%|██████████| 9/9 [00:00<00:00, 54.20batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 219\tTrain Loss = 0.1801\tValidation Loss = 8.1783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 220/500: 100%|██████████| 232/232 [00:09<00:00, 24.29batch/s, train_loss=0.178]\n",
      "Validation Epoch 220/500: 100%|██████████| 9/9 [00:00<00:00, 54.59batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 220\tTrain Loss = 0.1776\tValidation Loss = 8.1761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 221/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=0.173]\n",
      "Validation Epoch 221/500: 100%|██████████| 9/9 [00:00<00:00, 54.69batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 221\tTrain Loss = 0.1727\tValidation Loss = 8.1418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 222/500: 100%|██████████| 232/232 [00:09<00:00, 24.71batch/s, train_loss=0.174]\n",
      "Validation Epoch 222/500: 100%|██████████| 9/9 [00:00<00:00, 55.47batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 222\tTrain Loss = 0.1734\tValidation Loss = 8.1670\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 223/500: 100%|██████████| 232/232 [00:09<00:00, 24.68batch/s, train_loss=0.172]\n",
      "Validation Epoch 223/500: 100%|██████████| 9/9 [00:00<00:00, 53.41batch/s, val_loss=10.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 223\tTrain Loss = 0.1716\tValidation Loss = 8.2236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 224/500: 100%|██████████| 232/232 [00:09<00:00, 24.27batch/s, train_loss=0.172]\n",
      "Validation Epoch 224/500: 100%|██████████| 9/9 [00:00<00:00, 54.65batch/s, val_loss=10.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 224\tTrain Loss = 0.1714\tValidation Loss = 8.1332\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 225/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.173]\n",
      "Validation Epoch 225/500: 100%|██████████| 9/9 [00:00<00:00, 54.84batch/s, val_loss=10.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 225\tTrain Loss = 0.1726\tValidation Loss = 8.2436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 226/500:  56%|█████▌    | 130/232 [00:04<00:04, 24.59batch/s, train_loss=0.136]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 244/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.155]\n",
      "Validation Epoch 244/500: 100%|██████████| 9/9 [00:00<00:00, 54.64batch/s, val_loss=10.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 244\tTrain Loss = 0.1540\tValidation Loss = 8.3507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 245/500: 100%|██████████| 232/232 [00:09<00:00, 24.72batch/s, train_loss=0.154]\n",
      "Validation Epoch 245/500: 100%|██████████| 9/9 [00:00<00:00, 54.57batch/s, val_loss=10.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 245\tTrain Loss = 0.1545\tValidation Loss = 8.3398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 246/500: 100%|██████████| 232/232 [00:09<00:00, 24.76batch/s, train_loss=0.152]\n",
      "Validation Epoch 246/500: 100%|██████████| 9/9 [00:00<00:00, 54.11batch/s, val_loss=10.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 246\tTrain Loss = 0.1516\tValidation Loss = 8.3966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 247/500: 100%|██████████| 232/232 [00:09<00:00, 24.26batch/s, train_loss=0.157]\n",
      "Validation Epoch 247/500: 100%|██████████| 9/9 [00:00<00:00, 53.80batch/s, val_loss=10.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 247\tTrain Loss = 0.1561\tValidation Loss = 8.4069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 248/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.149]\n",
      "Validation Epoch 248/500: 100%|██████████| 9/9 [00:00<00:00, 56.20batch/s, val_loss=10.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 248\tTrain Loss = 0.1485\tValidation Loss = 8.3265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 249/500: 100%|██████████| 232/232 [00:09<00:00, 24.71batch/s, train_loss=0.15] \n",
      "Validation Epoch 249/500: 100%|██████████| 9/9 [00:00<00:00, 55.76batch/s, val_loss=10.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 249\tTrain Loss = 0.1496\tValidation Loss = 8.2717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 250/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=0.149]\n",
      "Validation Epoch 250/500: 100%|██████████| 9/9 [00:00<00:00, 51.71batch/s, val_loss=10.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 250\tTrain Loss = 0.1489\tValidation Loss = 8.3927\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 251/500: 100%|██████████| 232/232 [00:09<00:00, 24.72batch/s, train_loss=0.15] \n",
      "Validation Epoch 251/500: 100%|██████████| 9/9 [00:00<00:00, 54.38batch/s, val_loss=10.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 251\tTrain Loss = 0.1501\tValidation Loss = 8.3453\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 264/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.139]\n",
      "Validation Epoch 264/500: 100%|██████████| 9/9 [00:00<00:00, 56.05batch/s, val_loss=10.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 264\tTrain Loss = 0.1387\tValidation Loss = 8.3659\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 265/500: 100%|██████████| 232/232 [00:09<00:00, 24.78batch/s, train_loss=0.138] \n",
      "Validation Epoch 265/500: 100%|██████████| 9/9 [00:00<00:00, 55.08batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 265\tTrain Loss = 0.1369\tValidation Loss = 8.5184\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 266/500: 100%|██████████| 232/232 [00:09<00:00, 24.53batch/s, train_loss=0.137] \n",
      "Validation Epoch 266/500: 100%|██████████| 9/9 [00:00<00:00, 55.78batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 266\tTrain Loss = 0.1365\tValidation Loss = 8.4607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 267/500: 100%|██████████| 232/232 [00:09<00:00, 24.79batch/s, train_loss=0.138] \n",
      "Validation Epoch 267/500: 100%|██████████| 9/9 [00:00<00:00, 54.42batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 267\tTrain Loss = 0.1375\tValidation Loss = 8.4916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 268/500: 100%|██████████| 232/232 [00:09<00:00, 24.74batch/s, train_loss=0.138]\n",
      "Validation Epoch 268/500: 100%|██████████| 9/9 [00:00<00:00, 54.60batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 268\tTrain Loss = 0.1384\tValidation Loss = 8.4755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 269/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.135] \n",
      "Validation Epoch 269/500: 100%|██████████| 9/9 [00:00<00:00, 54.20batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 269\tTrain Loss = 0.1354\tValidation Loss = 8.5057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 270/500: 100%|██████████| 232/232 [00:09<00:00, 24.34batch/s, train_loss=0.135] \n",
      "Validation Epoch 270/500: 100%|██████████| 9/9 [00:00<00:00, 54.53batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 270\tTrain Loss = 0.1353\tValidation Loss = 8.5187\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 271/500: 100%|██████████| 232/232 [00:09<00:00, 24.76batch/s, train_loss=0.136] \n",
      "Validation Epoch 271/500: 100%|██████████| 9/9 [00:00<00:00, 54.79batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 271\tTrain Loss = 0.1352\tValidation Loss = 8.5154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 272/500: 100%|██████████| 232/232 [00:09<00:00, 24.61batch/s, train_loss=0.136]\n",
      "Validation Epoch 272/500: 100%|██████████| 9/9 [00:00<00:00, 54.29batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 272\tTrain Loss = 0.1354\tValidation Loss = 8.4727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 273/500: 100%|██████████| 232/232 [00:09<00:00, 24.50batch/s, train_loss=0.135] \n",
      "Validation Epoch 273/500: 100%|██████████| 9/9 [00:00<00:00, 54.70batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 273\tTrain Loss = 0.1346\tValidation Loss = 8.5492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 293/500: 100%|██████████| 232/232 [00:09<00:00, 24.50batch/s, train_loss=0.123] \n",
      "Validation Epoch 293/500: 100%|██████████| 9/9 [00:00<00:00, 54.03batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 293\tTrain Loss = 0.1225\tValidation Loss = 8.5614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 294/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.124] \n",
      "Validation Epoch 294/500: 100%|██████████| 9/9 [00:00<00:00, 54.64batch/s, val_loss=11.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 294\tTrain Loss = 0.1233\tValidation Loss = 8.6524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 295/500: 100%|██████████| 232/232 [00:09<00:00, 24.75batch/s, train_loss=0.122] \n",
      "Validation Epoch 295/500: 100%|██████████| 9/9 [00:00<00:00, 55.83batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 295\tTrain Loss = 0.1218\tValidation Loss = 8.5520\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 296/500: 100%|██████████| 232/232 [00:09<00:00, 24.52batch/s, train_loss=0.121] \n",
      "Validation Epoch 296/500: 100%|██████████| 9/9 [00:00<00:00, 55.05batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 296\tTrain Loss = 0.1211\tValidation Loss = 8.5507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 297/500: 100%|██████████| 232/232 [00:09<00:00, 24.74batch/s, train_loss=0.121] \n",
      "Validation Epoch 297/500: 100%|██████████| 9/9 [00:00<00:00, 54.93batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 297\tTrain Loss = 0.1210\tValidation Loss = 8.5538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 298/500: 100%|██████████| 232/232 [00:09<00:00, 24.79batch/s, train_loss=0.121] \n",
      "Validation Epoch 298/500: 100%|██████████| 9/9 [00:00<00:00, 54.69batch/s, val_loss=10.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 298\tTrain Loss = 0.1209\tValidation Loss = 8.5135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 299/500: 100%|██████████| 232/232 [00:09<00:00, 24.64batch/s, train_loss=0.121] \n",
      "Validation Epoch 299/500: 100%|██████████| 9/9 [00:00<00:00, 55.09batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 299\tTrain Loss = 0.1208\tValidation Loss = 8.5487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 300/500: 100%|██████████| 232/232 [00:09<00:00, 24.73batch/s, train_loss=0.121] \n",
      "Validation Epoch 300/500: 100%|██████████| 9/9 [00:00<00:00, 55.65batch/s, val_loss=11]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 300\tTrain Loss = 0.1203\tValidation Loss = 8.5221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 301/500: 100%|██████████| 232/232 [00:09<00:00, 24.82batch/s, train_loss=0.119] \n",
      "Validation Epoch 301/500: 100%|██████████| 9/9 [00:00<00:00, 55.51batch/s, val_loss=11.1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 301\tTrain Loss = 0.1182\tValidation Loss = 8.6586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 302/500:  87%|████████▋ | 202/232 [00:07<00:01, 21.27batch/s, train_loss=0.111] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 320/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.114] \n",
      "Validation Epoch 320/500: 100%|██████████| 9/9 [00:00<00:00, 54.93batch/s, val_loss=11.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 320\tTrain Loss = 0.1134\tValidation Loss = 8.7129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 321/500: 100%|██████████| 232/232 [00:09<00:00, 24.78batch/s, train_loss=0.11]  \n",
      "Validation Epoch 321/500: 100%|██████████| 9/9 [00:00<00:00, 55.57batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 321\tTrain Loss = 0.1094\tValidation Loss = 8.7579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 322/500: 100%|██████████| 232/232 [00:09<00:00, 24.49batch/s, train_loss=0.111] \n",
      "Validation Epoch 322/500: 100%|██████████| 9/9 [00:00<00:00, 55.44batch/s, val_loss=11.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 322\tTrain Loss = 0.1102\tValidation Loss = 8.7222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 323/500: 100%|██████████| 232/232 [00:09<00:00, 24.67batch/s, train_loss=0.11]  \n",
      "Validation Epoch 323/500: 100%|██████████| 9/9 [00:00<00:00, 56.00batch/s, val_loss=11.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 323\tTrain Loss = 0.1100\tValidation Loss = 8.7295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 324/500: 100%|██████████| 232/232 [00:09<00:00, 24.79batch/s, train_loss=0.109] \n",
      "Validation Epoch 324/500: 100%|██████████| 9/9 [00:00<00:00, 55.35batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 324\tTrain Loss = 0.1088\tValidation Loss = 8.8153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 325/500: 100%|██████████| 232/232 [00:09<00:00, 24.70batch/s, train_loss=0.11]  \n",
      "Validation Epoch 325/500: 100%|██████████| 9/9 [00:00<00:00, 52.23batch/s, val_loss=11.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 325\tTrain Loss = 0.1099\tValidation Loss = 8.7498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 326/500: 100%|██████████| 232/232 [00:09<00:00, 24.54batch/s, train_loss=0.108] \n",
      "Validation Epoch 326/500: 100%|██████████| 9/9 [00:00<00:00, 54.81batch/s, val_loss=11.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 326\tTrain Loss = 0.1075\tValidation Loss = 8.7314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 327/500: 100%|██████████| 232/232 [00:09<00:00, 24.78batch/s, train_loss=0.109] \n",
      "Validation Epoch 327/500: 100%|██████████| 9/9 [00:00<00:00, 55.07batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 327\tTrain Loss = 0.1086\tValidation Loss = 8.7959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 328/500: 100%|██████████| 232/232 [00:09<00:00, 24.75batch/s, train_loss=0.109] \n",
      "Validation Epoch 328/500: 100%|██████████| 9/9 [00:00<00:00, 55.23batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 328\tTrain Loss = 0.1092\tValidation Loss = 8.8179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 329/500: 100%|██████████| 232/232 [00:09<00:00, 24.50batch/s, train_loss=0.11]  \n",
      "Validation Epoch 329/500:   0%|          | 0/9 [00:00<?, ?batch/s, val_loss=25.1]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 346/500: 100%|██████████| 232/232 [00:09<00:00, 24.81batch/s, train_loss=0.102] \n",
      "Validation Epoch 346/500: 100%|██████████| 9/9 [00:00<00:00, 55.52batch/s, val_loss=11.4]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 346\tTrain Loss = 0.1020\tValidation Loss = 8.8343\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 347/500: 100%|██████████| 232/232 [00:09<00:00, 24.87batch/s, train_loss=0.103] \n",
      "Validation Epoch 347/500: 100%|██████████| 9/9 [00:00<00:00, 54.74batch/s, val_loss=11.5]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 347\tTrain Loss = 0.1026\tValidation Loss = 8.9175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 348/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.101] \n",
      "Validation Epoch 348/500: 100%|██████████| 9/9 [00:00<00:00, 54.31batch/s, val_loss=11.4]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 348\tTrain Loss = 0.1005\tValidation Loss = 8.8996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 349/500: 100%|██████████| 232/232 [00:09<00:00, 24.48batch/s, train_loss=0.0988]\n",
      "Validation Epoch 349/500: 100%|██████████| 9/9 [00:00<00:00, 53.38batch/s, val_loss=11.4]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 349\tTrain Loss = 0.0988\tValidation Loss = 8.8347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 350/500: 100%|██████████| 232/232 [00:09<00:00, 24.70batch/s, train_loss=0.101] \n",
      "Validation Epoch 350/500: 100%|██████████| 9/9 [00:00<00:00, 55.27batch/s, val_loss=11.4]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 350\tTrain Loss = 0.1004\tValidation Loss = 8.8768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 351/500: 100%|██████████| 232/232 [00:09<00:00, 24.75batch/s, train_loss=0.101] \n",
      "Validation Epoch 351/500: 100%|██████████| 9/9 [00:00<00:00, 55.19batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 351\tTrain Loss = 0.1006\tValidation Loss = 8.8172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 352/500: 100%|██████████| 232/232 [00:09<00:00, 24.59batch/s, train_loss=0.0991]\n",
      "Validation Epoch 352/500: 100%|██████████| 9/9 [00:00<00:00, 55.57batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 352\tTrain Loss = 0.0987\tValidation Loss = 8.7557\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 353/500: 100%|██████████| 232/232 [00:09<00:00, 24.79batch/s, train_loss=0.0984]\n",
      "Validation Epoch 353/500: 100%|██████████| 9/9 [00:00<00:00, 55.41batch/s, val_loss=11.3]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 353\tTrain Loss = 0.0980\tValidation Loss = 8.8205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 354/500: 100%|██████████| 232/232 [00:09<00:00, 24.75batch/s, train_loss=0.098] \n",
      "Validation Epoch 354/500: 100%|██████████| 9/9 [00:00<00:00, 54.45batch/s, val_loss=11.4]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 354\tTrain Loss = 0.0975\tValidation Loss = 8.8625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 355/500:  68%|██████▊   | 158/232 [00:05<00:03, 23.70batch/s, train_loss=0.0828]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 407/500: 100%|██████████| 232/232 [00:09<00:00, 24.69batch/s, train_loss=0.0841]\n",
      "Validation Epoch 407/500: 100%|██████████| 9/9 [00:00<00:00, 54.32batch/s, val_loss=11.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 407\tTrain Loss = 0.0837\tValidation Loss = 9.0827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 408/500: 100%|██████████| 232/232 [00:09<00:00, 24.67batch/s, train_loss=0.0853]\n",
      "Validation Epoch 408/500: 100%|██████████| 9/9 [00:00<00:00, 54.64batch/s, val_loss=11.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 408\tTrain Loss = 0.0849\tValidation Loss = 9.2258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 409/500: 100%|██████████| 232/232 [00:09<00:00, 24.89batch/s, train_loss=0.0856]\n",
      "Validation Epoch 409/500: 100%|██████████| 9/9 [00:00<00:00, 55.33batch/s, val_loss=11.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 409\tTrain Loss = 0.0852\tValidation Loss = 9.0251\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 410/500: 100%|██████████| 232/232 [00:09<00:00, 24.76batch/s, train_loss=0.0851]\n",
      "Validation Epoch 410/500: 100%|██████████| 9/9 [00:00<00:00, 55.44batch/s, val_loss=11.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 410\tTrain Loss = 0.0848\tValidation Loss = 9.0839\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 411/500: 100%|██████████| 232/232 [00:09<00:00, 24.59batch/s, train_loss=0.0828]\n",
      "Validation Epoch 411/500: 100%|██████████| 9/9 [00:00<00:00, 50.71batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 411\tTrain Loss = 0.0828\tValidation Loss = 9.1490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 412/500: 100%|██████████| 232/232 [00:09<00:00, 24.70batch/s, train_loss=0.0824]\n",
      "Validation Epoch 412/500: 100%|██████████| 9/9 [00:00<00:00, 54.79batch/s, val_loss=11.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 412\tTrain Loss = 0.0824\tValidation Loss = 9.0389\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 413/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.084] \n",
      "Validation Epoch 413/500: 100%|██████████| 9/9 [00:00<00:00, 55.21batch/s, val_loss=11.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 413\tTrain Loss = 0.0840\tValidation Loss = 9.0672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 414/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.0852]\n",
      "Validation Epoch 414/500: 100%|██████████| 9/9 [00:00<00:00, 54.86batch/s, val_loss=11.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 414\tTrain Loss = 0.0852\tValidation Loss = 9.0205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 415/500: 100%|██████████| 232/232 [00:09<00:00, 24.37batch/s, train_loss=0.0844]\n",
      "Validation Epoch 415/500: 100%|██████████| 9/9 [00:00<00:00, 55.40batch/s, val_loss=11.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 415\tTrain Loss = 0.0840\tValidation Loss = 9.0761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 416/500:  82%|████████▏ | 190/232 [00:07<00:01, 21.89batch/s, train_loss=0.0744]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 444/500: 100%|██████████| 232/232 [00:09<00:00, 24.45batch/s, train_loss=0.0769]\n",
      "Validation Epoch 444/500: 100%|██████████| 9/9 [00:00<00:00, 55.36batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 444\tTrain Loss = 0.0766\tValidation Loss = 9.2086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 445/500: 100%|██████████| 232/232 [00:09<00:00, 24.81batch/s, train_loss=0.0772]\n",
      "Validation Epoch 445/500: 100%|██████████| 9/9 [00:00<00:00, 53.61batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 445\tTrain Loss = 0.0772\tValidation Loss = 9.2086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 446/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.0776]\n",
      "Validation Epoch 446/500: 100%|██████████| 9/9 [00:00<00:00, 53.49batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 446\tTrain Loss = 0.0776\tValidation Loss = 9.2073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 447/500: 100%|██████████| 232/232 [00:09<00:00, 24.63batch/s, train_loss=0.0778]\n",
      "Validation Epoch 447/500: 100%|██████████| 9/9 [00:00<00:00, 54.25batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 447\tTrain Loss = 0.0774\tValidation Loss = 9.1889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 448/500: 100%|██████████| 232/232 [00:09<00:00, 24.28batch/s, train_loss=0.0776]\n",
      "Validation Epoch 448/500: 100%|██████████| 9/9 [00:00<00:00, 55.43batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 448\tTrain Loss = 0.0773\tValidation Loss = 9.1686\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 449/500: 100%|██████████| 232/232 [00:09<00:00, 24.73batch/s, train_loss=0.0777]\n",
      "Validation Epoch 449/500: 100%|██████████| 9/9 [00:00<00:00, 55.17batch/s, val_loss=11.7]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 449\tTrain Loss = 0.0777\tValidation Loss = 9.1087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 450/500: 100%|██████████| 232/232 [00:09<00:00, 24.73batch/s, train_loss=0.0752]\n",
      "Validation Epoch 450/500: 100%|██████████| 9/9 [00:00<00:00, 53.62batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 450\tTrain Loss = 0.0749\tValidation Loss = 9.1905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 451/500: 100%|██████████| 232/232 [00:09<00:00, 24.47batch/s, train_loss=0.0752]\n",
      "Validation Epoch 451/500: 100%|██████████| 9/9 [00:00<00:00, 55.62batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 451\tTrain Loss = 0.0749\tValidation Loss = 9.2085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 452/500: 100%|██████████| 232/232 [00:09<00:00, 24.81batch/s, train_loss=0.0749]\n",
      "Validation Epoch 452/500: 100%|██████████| 9/9 [00:00<00:00, 54.72batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 452\tTrain Loss = 0.0746\tValidation Loss = 9.1979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 453/500: 100%|██████████| 232/232 [00:09<00:00, 24.85batch/s, train_loss=0.0789]\n",
      "Validation Epoch 453/500: 100%|██████████| 9/9 [00:00<00:00, 55.22batch/s, val_loss=11.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 453\tTrain Loss = 0.0786\tValidation Loss = 9.2443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 454/500:   7%|▋         | 16/232 [00:00<00:06, 33.81batch/s, train_loss=0.0508]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Training Epoch 483/500: 100%|██████████| 232/232 [00:09<00:00, 24.77batch/s, train_loss=0.0716]\n",
      "Validation Epoch 483/500: 100%|██████████| 9/9 [00:00<00:00, 56.11batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 483\tTrain Loss = 0.0713\tValidation Loss = 9.1955\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 484/500: 100%|██████████| 232/232 [00:09<00:00, 24.61batch/s, train_loss=0.07]  \n",
      "Validation Epoch 484/500: 100%|██████████| 9/9 [00:00<00:00, 54.92batch/s, val_loss=11.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 484\tTrain Loss = 0.0697\tValidation Loss = 9.2282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 485/500: 100%|██████████| 232/232 [00:09<00:00, 24.83batch/s, train_loss=0.07]  \n",
      "Validation Epoch 485/500: 100%|██████████| 9/9 [00:00<00:00, 54.65batch/s, val_loss=11.8]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 485\tTrain Loss = 0.0700\tValidation Loss = 9.1738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 486/500: 100%|██████████| 232/232 [00:09<00:00, 24.76batch/s, train_loss=0.0707]\n",
      "Validation Epoch 486/500: 100%|██████████| 9/9 [00:00<00:00, 55.82batch/s, val_loss=11.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 486\tTrain Loss = 0.0704\tValidation Loss = 9.2446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 487/500: 100%|██████████| 232/232 [00:09<00:00, 24.56batch/s, train_loss=0.0707]\n",
      "Validation Epoch 487/500: 100%|██████████| 9/9 [00:00<00:00, 55.61batch/s, val_loss=12]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 487\tTrain Loss = 0.0704\tValidation Loss = 9.3114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 488/500: 100%|██████████| 232/232 [00:09<00:00, 24.83batch/s, train_loss=0.0702]\n",
      "Validation Epoch 488/500: 100%|██████████| 9/9 [00:00<00:00, 53.40batch/s, val_loss=12]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 488\tTrain Loss = 0.0699\tValidation Loss = 9.3089\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 489/500: 100%|██████████| 232/232 [00:09<00:00, 24.80batch/s, train_loss=0.0685]\n",
      "Validation Epoch 489/500: 100%|██████████| 9/9 [00:00<00:00, 56.29batch/s, val_loss=12]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 489\tTrain Loss = 0.0685\tValidation Loss = 9.3539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 490/500: 100%|██████████| 232/232 [00:09<00:00, 24.74batch/s, train_loss=0.0696]\n",
      "Validation Epoch 490/500: 100%|██████████| 9/9 [00:00<00:00, 45.34batch/s, val_loss=14]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 490\tTrain Loss = 0.0696\tValidation Loss = 9.3433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 491/500: 100%|██████████| 232/232 [00:09<00:00, 24.60batch/s, train_loss=0.0705]\n",
      "Validation Epoch 491/500: 100%|██████████| 9/9 [00:00<00:00, 55.58batch/s, val_loss=11.9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 491\tTrain Loss = 0.0705\tValidation Loss = 9.2481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 492/500:  28%|██▊       | 66/232 [00:02<00:05, 28.52batch/s, train_loss=0.0479]"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "# Initialize loss lists to track training and validation loss\n",
    "t_loss = []\n",
    "ev_loss = []\n",
    "\n",
    "# Start training loop\n",
    "for epoch in range(var.TRAIN['epoch']):\n",
    "    epoch_train_loss = 0\n",
    "    epoch_val_loss = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    # Training loop with progress bar\n",
    "    with tqdm(train_dataloader, desc=f\"Training Epoch {epoch+1}/{var.TRAIN['epoch']}\", unit=\"batch\") as train_bar:\n",
    "        for batch in train_bar:\n",
    "            optim.zero_grad()\n",
    "            \n",
    "            src, src_mask, trg, trg_mask = batch\n",
    "            src, src_mask, trg, trg_mask = src.to(var.device), src_mask.to(var.device), trg.to(var.device), trg_mask.to(var.device)\n",
    "            \n",
    "            output, _ = model(src, trg[:, :-1], src_mask, trg_mask[:, :-1])\n",
    "            \n",
    "            output_dim = output.shape[-1]\n",
    "            output = output.contiguous().view(-1, output_dim)\n",
    "            trg = trg[:, 1:].contiguous().view(-1)\n",
    "            \n",
    "            loss = criterion(output, trg)\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "            \n",
    "            epoch_train_loss += loss.item()\n",
    "            \n",
    "            # Update progress bar description with current loss\n",
    "            train_bar.set_postfix(train_loss=epoch_train_loss / (train_bar.n + 1))\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    # Validation loop with progress bar\n",
    "    with tqdm(test_dataloader, desc=f\"Validation Epoch {epoch+1}/{var.TRAIN['epoch']}\", unit=\"batch\") as val_bar:\n",
    "        for batch in val_bar:\n",
    "            src, src_mask, trg, trg_mask = batch\n",
    "            src, src_mask, trg, trg_mask = src.to(var.device), src_mask.to(var.device), trg.to(var.device), trg_mask.to(var.device)\n",
    "            \n",
    "            output, _ = model(src, trg[:, :-1], src_mask, trg_mask[:, :-1])\n",
    "            \n",
    "            output_dim = output.shape[-1]\n",
    "            output = output.contiguous().view(-1, output_dim)\n",
    "            trg = trg[:, 1:].contiguous().view(-1)\n",
    "            \n",
    "            loss = criterion(output, trg)\n",
    "            epoch_val_loss += loss.item()\n",
    "            \n",
    "            # Update progress bar description with current loss\n",
    "            val_bar.set_postfix(val_loss=epoch_val_loss / (val_bar.n + 1))\n",
    "    \n",
    "    # Average losses for the epoch\n",
    "    epoch_train_loss /= len(train_dataloader)\n",
    "    epoch_val_loss /= len(test_dataloader)\n",
    "    \n",
    "    t_loss.append(epoch_train_loss)\n",
    "    ev_loss.append(epoch_val_loss)\n",
    "    \n",
    "    # Print summary of losses for this epoch\n",
    "    print(f\"Epoch: {epoch+1}\\tTrain Loss = {epoch_train_loss:.4f}\\tValidation Loss = {epoch_val_loss:.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:01:51.977295Z",
     "iopub.status.busy": "2024-11-23T14:01:51.976509Z",
     "iopub.status.idle": "2024-11-23T14:02:00.749513Z",
     "shell.execute_reply": "2024-11-23T14:02:00.748502Z",
     "shell.execute_reply.started": "2024-11-23T14:01:51.977268Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (3.2.4)\n",
      "Requirement already satisfied: rouge-score in /opt/conda/lib/python3.10/site-packages (0.1.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from nltk) (1.16.0)\n",
      "Requirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.4.0)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.26.4)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install nltk rouge-score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:09:30.660913Z",
     "iopub.status.busy": "2024-11-23T14:09:30.660039Z",
     "iopub.status.idle": "2024-11-23T14:09:30.666116Z",
     "shell.execute_reply": "2024-11-23T14:09:30.665178Z",
     "shell.execute_reply.started": "2024-11-23T14:09:30.660877Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU Score: 0.4523\n",
      "Average ROUGE Score: 0.5128\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "# Initialize BLEU and ROUGE scorers\n",
    "bleu_scores = []\n",
    "rouge_scorer = rouge_scorer.RougeScorer([\"rouge1\", \"rouge2\", \"rougeL\"], use_stemmer=True)\n",
    "\n",
    "# Evaluate model on the test set\n",
    "model.eval()\n",
    "references = []  # To store reference translations (target sentences)\n",
    "hypotheses = []  # To store model predictions\n",
    "\n",
    "with tqdm(test_dataloader, desc=\"Testing\", unit=\"batch\") as test_bar:\n",
    "    for batch in test_bar:\n",
    "        src, src_mask, trg, trg_mask = batch\n",
    "        src, src_mask, trg, trg_mask = src.to(var.device), src_mask.to(var.device), trg.to(var.device), trg_mask.to(var.device)\n",
    "        \n",
    "        # Get model output (skip the last token for prediction)\n",
    "        with torch.no_grad():\n",
    "            output, _ = model(src, trg[:, :-1], src_mask, trg_mask[:, :-1])\n",
    "        \n",
    "        # Convert predictions and target to readable form\n",
    "        output_dim = output.shape[-1]\n",
    "        output = output.contiguous().view(-1, output_dim)\n",
    "        pred_tokens = output.argmax(dim=-1)  # Predicted tokens (argmax to get the most likely words)\n",
    "        trg = trg[:, 1:].contiguous().view(-1)  # The reference (ground truth) target sentence\n",
    "        \n",
    "        # Store hypotheses and references for BLEU and ROUGE\n",
    "        for i in range(len(pred_tokens)):\n",
    "            pred_sentence = pred_tokens[i].cpu().numpy()  # Convert to NumPy for further processing\n",
    "            trg_sentence = trg[i].cpu().numpy()\n",
    "\n",
    "            # Convert token IDs to words (this will depend on how your tokenizer works)\n",
    "            pred_sentence_words = tokenizer.decode(pred_sentence)  # Use your tokenizer here\n",
    "            trg_sentence_words = tokenizer.decode(trg_sentence)  # Use your tokenizer here\n",
    "\n",
    "            hypotheses.append(pred_sentence_words)\n",
    "            references.append([trg_sentence_words])\n",
    "\n",
    "            # ROUGE computation\n",
    "            scores = rouge_scorer.score(trg_sentence_words, pred_sentence_words)\n",
    "            rouge_scores = {key: value.fmeasure for key, value in scores.items()}\n",
    "            rouge_score = rouge_scores['rougeL']\n",
    "            bleu_scores.append(rouge_score)\n",
    "        \n",
    "        test_bar.set_postfix(bleu_score=corpus_bleu(references, hypotheses))\n",
    "        \n",
    "# Calculate BLEU score\n",
    "bleu = corpus_bleu(references, hypotheses)\n",
    "\n",
    "# Calculate average ROUGE score\n",
    "average_rouge = sum(bleu_scores) / len(bleu_scores)\n",
    "\n",
    "# Print final evaluation scores\n",
    "print(f\"BLEU Score: {bleu:.4f}\")\n",
    "print(f\"Average ROUGE Score: {average_rouge:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:02:00.757695Z",
     "iopub.status.busy": "2024-11-23T14:02:00.757461Z",
     "iopub.status.idle": "2024-11-23T14:02:01.011524Z",
     "shell.execute_reply": "2024-11-23T14:02:01.010432Z",
     "shell.execute_reply.started": "2024-11-23T14:02:00.757669Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHHCAYAAACRAnNyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABtbUlEQVR4nO3dd3gU5d7G8e/uJtn0QkIn9NCrNAEpKkoTAbEhKmDBAii2ox4VwYa+No7owQ5YEMsBRAEREFEp0pv03jvpfXfeP4ZsWBJaSDIp9+e69sq03fntGM3tM8/zjM0wDAMRERGRIshudQEiIiIi56KgIiIiIkWWgoqIiIgUWQoqIiIiUmQpqIiIiEiRpaAiIiIiRZaCioiIiBRZCioiIiJSZCmoiIiISJGloCKSR4MGDaJ69ep5eu+oUaOw2Wz5W1ARs3v3bmw2GxMnTiz0c9tsNkaNGuVZnzhxIjabjd27d1/wvdWrV2fQoEH5Ws/l/K6IlHYKKlLi2Gy2i3r9/vvvVpda6j3yyCPYbDa2b99+zmOee+45bDYb69atK8TKLt3BgwcZNWoUa9assboUj6yw+NZbb1ldikie+VhdgEh++/LLL73Wv/jiC+bOnZtje/369S/rPJ988glutztP733++ed55plnLuv8JcGAAQMYN24ckydPZuTIkbke880339C4cWOaNGmS5/Pcdddd3H777Tidzjx/xoUcPHiQ0aNHU716dZo1a+a173J+V0RKOwUVKXHuvPNOr/WlS5cyd+7cHNvPlpycTGBg4EWfx9fXN0/1Afj4+ODjo3/92rRpQ+3atfnmm29yDSpLlixh165dvP7665d1HofDgcPhuKzPuByX87siUtrp1o+USp07d6ZRo0asXLmSjh07EhgYyL///W8AfvzxR3r27EmlSpVwOp3UqlWLl19+GZfL5fUZZ/c7OLOZ/eOPP6ZWrVo4nU5atWrF8uXLvd6bWx8Vm83GsGHDmD59Oo0aNcLpdNKwYUN++eWXHPX//vvvtGzZEn9/f2rVqsVHH3100f1e/vzzT2655RaqVq2K0+kkOjqaxx57jJSUlBzfLzg4mAMHDtCnTx+Cg4MpW7YsTz75ZI5rERsby6BBgwgLCyM8PJyBAwcSGxt7wVrAbFXZvHkzq1atyrFv8uTJ2Gw2+vfvT3p6OiNHjqRFixaEhYURFBREhw4dWLBgwQXPkVsfFcMweOWVV6hSpQqBgYFcffXV/PPPPznee/LkSZ588kkaN25McHAwoaGhdO/enbVr13qO+f3332nVqhUAgwcP9txezOqfk1sflaSkJJ544gmio6NxOp3UrVuXt956i7MfaH8pvxd5dfToUe69917Kly+Pv78/TZs2ZdKkSTmOmzJlCi1atCAkJITQ0FAaN27Mf/7zH8/+jIwMRo8eTUxMDP7+/kRGRnLVVVcxd+7cfKtVSh/9L52UWidOnKB79+7cfvvt3HnnnZQvXx4w/6gFBwfz+OOPExwczG+//cbIkSOJj4/nzTffvODnTp48mYSEBB544AFsNhv/93//x0033cTOnTsv+H/Wf/31F1OnTuXhhx8mJCSE9957j379+rF3714iIyMBWL16Nd26daNixYqMHj0al8vFSy+9RNmyZS/qe3///fckJyfz0EMPERkZybJlyxg3bhz79+/n+++/9zrW5XLRtWtX2rRpw1tvvcW8efN4++23qVWrFg899BBg/sHv3bs3f/31Fw8++CD169dn2rRpDBw48KLqGTBgAKNHj2by5MlcccUVXuf+7rvv6NChA1WrVuX48eN8+umn9O/fn/vvv5+EhAQ+++wzunbtyrJly3LcbrmQkSNH8sorr9CjRw969OjBqlWruP7660lPT/c6bufOnUyfPp1bbrmFGjVqcOTIET766CM6derExo0bqVSpEvXr1+ell15i5MiRDBkyhA4dOgDQrl27XM9tGAY33ngjCxYs4N5776VZs2bMmTOHp556igMHDvDuu+96HX8xvxd5lZKSQufOndm+fTvDhg2jRo0afP/99wwaNIjY2FgeffRRAObOnUv//v259tpreeONNwDYtGkTixYt8hwzatQoxowZw3333Ufr1q2Jj49nxYoVrFq1iuuuu+6y6pRSzBAp4YYOHWqc/aveqVMnAzA+/PDDHMcnJyfn2PbAAw8YgYGBRmpqqmfbwIEDjWrVqnnWd+3aZQBGZGSkcfLkSc/2H3/80QCMn376ybPtxRdfzFETYPj5+Rnbt2/3bFu7dq0BGOPGjfNs69WrlxEYGGgcOHDAs23btm2Gj49Pjs/MTW7fb8yYMYbNZjP27Nnj9f0A46WXXvI6tnnz5kaLFi0869OnTzcA4//+7/882zIzM40OHToYgDFhwoQL1tSqVSujSpUqhsvl8mz75ZdfDMD46KOPPJ+Zlpbm9b5Tp04Z5cuXN+655x6v7YDx4osvetYnTJhgAMauXbsMwzCMo0ePGn5+fkbPnj0Nt9vtOe7f//63ARgDBw70bEtNTfWqyzDMf9ZOp9Pr2ixfvvyc3/fs35Wsa/bKK694HXfzzTcbNpvN63fgYn8vcpP1O/nmm2+e85ixY8cagPHVV195tqWnpxtt27Y1goODjfj4eMMwDOPRRx81QkNDjczMzHN+VtOmTY2ePXuetyaRS6VbP1JqOZ1OBg8enGN7QECAZzkhIYHjx4/ToUMHkpOT2bx58wU/97bbbiMiIsKznvV/1zt37rzge7t06UKtWrU8602aNCE0NNTzXpfLxbx58+jTpw+VKlXyHFe7dm26d+9+wc8H7++XlJTE8ePHadeuHYZhsHr16hzHP/jgg17rHTp08Pous2bNwsfHx9PCAmafkOHDh19UPWD2K9q/fz9//PGHZ9vkyZPx8/Pjlltu8Xymn58fAG63m5MnT5KZmUnLli1zvW10PvPmzSM9PZ3hw4d73S4bMWJEjmOdTid2u/mfSpfLxYkTJwgODqZu3bqXfN4ss2bNwuFw8Mgjj3htf+KJJzAMg9mzZ3ttv9DvxeWYNWsWFSpUoH///p5tvr6+PPLIIyQmJrJw4UIAwsPDSUpKOu9tnPDwcP755x+2bdt22XWJZFFQkVKrcuXKnj98Z/rnn3/o27cvYWFhhIaGUrZsWU9H3Li4uAt+btWqVb3Ws0LLqVOnLvm9We/Peu/Ro0dJSUmhdu3aOY7LbVtu9u7dy6BBgyhTpoyn30mnTp2AnN/P398/xy2lM+sB2LNnDxUrViQ4ONjruLp1615UPQC33347DoeDyZMnA5Camsq0adPo3r27V+ibNGkSTZo08fR/KFu2LDNnzryofy5n2rNnDwAxMTFe28uWLet1PjBD0bvvvktMTAxOp5OoqCjKli3LunXrLvm8Z56/UqVKhISEeG3PGomWVV+WC/1eXI49e/YQExPjCWPnquXhhx+mTp06dO/enSpVqnDPPffk6Cfz0ksvERsbS506dWjcuDFPPfVUkR9WLkWfgoqUWme2LGSJjY2lU6dOrF27lpdeeomffvqJuXPneu7JX8wQ03ONLjHO6iSZ3++9GC6Xi+uuu46ZM2fy9NNPM336dObOnevp9Hn29yuskTLlypXjuuuu43//+x8ZGRn89NNPJCQkMGDAAM8xX331FYMGDaJWrVp89tln/PLLL8ydO5drrrmmQIf+vvbaazz++ON07NiRr776ijlz5jB37lwaNmxYaEOOC/r34mKUK1eONWvWMGPGDE//mu7du3v1RerYsSM7duzg888/p1GjRnz66adcccUVfPrpp4VWp5Q86kwrcobff/+dEydOMHXqVDp27OjZvmvXLgurylauXDn8/f1znSDtfJOmZVm/fj1bt25l0qRJ3H333Z7tlzMqo1q1asyfP5/ExESvVpUtW7Zc0ucMGDCAX375hdmzZzN58mRCQ0Pp1auXZ/8PP/xAzZo1mTp1qtftmhdffDFPNQNs27aNmjVrerYfO3YsRyvFDz/8wNVXX81nn33mtT02NpaoqCjP+qXMNFytWjXmzZtHQkKCV6tK1q3FrPoKQ7Vq1Vi3bh1ut9urVSW3Wvz8/OjVqxe9evXC7Xbz8MMP89FHH/HCCy94WvTKlCnD4MGDGTx4MImJiXTs2JFRo0Zx3333Fdp3kpJFLSoiZ8j6P9cz/081PT2d//73v1aV5MXhcNClSxemT5/OwYMHPdu3b9+eo1/Dud4P3t/PMAyvIaaXqkePHmRmZjJ+/HjPNpfLxbhx4y7pc/r06UNgYCD//e9/mT17NjfddBP+/v7nrf3vv/9myZIll1xzly5d8PX1Zdy4cV6fN3bs2BzHOhyOHC0X33//PQcOHPDaFhQUBHBRw7J79OiBy+Xi/fff99r+7rvvYrPZLrq/UX7o0aMHhw8f5ttvv/Vsy8zMZNy4cQQHB3tuC544ccLrfXa73TMJX1paWq7HBAcHU7t2bc9+kbxQi4rIGdq1a0dERAQDBw70TO/+5ZdfFmoT+4WMGjWKX3/9lfbt2/PQQw95/uA1atTogtO316tXj1q1avHkk09y4MABQkND+d///ndZfR169epF+/bteeaZZ9i9ezcNGjRg6tSpl9x/Izg4mD59+nj6qZx52wfghhtuYOrUqfTt25eePXuya9cuPvzwQxo0aEBiYuIlnStrPpgxY8Zwww030KNHD1avXs3s2bO9WkmyzvvSSy8xePBg2rVrx/r16/n666+9WmIAatWqRXh4OB9++CEhISEEBQXRpk0batSokeP8vXr14uqrr+a5555j9+7dNG3alF9//ZUff/yRESNGeHWczQ/z588nNTU1x/Y+ffowZMgQPvroIwYNGsTKlSupXr06P/zwA4sWLWLs2LGeFp/77ruPkydPcs0111ClShX27NnDuHHjaNasmac/S4MGDejcuTMtWrSgTJkyrFixgh9++IFhw4bl6/eRUsaawUYihedcw5MbNmyY6/GLFi0yrrzySiMgIMCoVKmS8a9//cuYM2eOARgLFizwHHeu4cm5DQXlrOGy5xqePHTo0BzvrVatmtdwWcMwjPnz5xvNmzc3/Pz8jFq1ahmffvqp8cQTTxj+/v7nuArZNm7caHTp0sUIDg42oqKijPvvv98z3PXMobUDBw40goKCcrw/t9pPnDhh3HXXXUZoaKgRFhZm3HXXXcbq1asvenhylpkzZxqAUbFixRxDgt1ut/Haa68Z1apVM5xOp9G8eXPj559/zvHPwTAuPDzZMAzD5XIZo0ePNipWrGgEBAQYnTt3NjZs2JDjeqemphpPPPGE57j27dsbS5YsMTp16mR06tTJ67w//vij0aBBA89Q8azvnluNCQkJxmOPPWZUqlTJ8PX1NWJiYow333zTa7h01ne52N+Ls2X9Tp7r9eWXXxqGYRhHjhwxBg8ebERFRRl+fn5G48aNc/xz++GHH4zrr7/eKFeunOHn52dUrVrVeOCBB4xDhw55jnnllVeM1q1bG+Hh4UZAQIBRr14949VXXzXS09PPW6fI+dgMowj9r6KI5FmfPn00NFREShz1UREphs6e7n7btm3MmjWLzp07W1OQiEgBUYuKSDFUsWJFBg0aRM2aNdmzZw/jx48nLS2N1atX55gbRESkOFNnWpFiqFu3bnzzzTccPnwYp9NJ27Ztee211xRSRKTEUYuKiIiIFFnqoyIiIiJFloKKiIiIFFnFuo+K2+3m4MGDhISEXNL01SIiImIdwzBISEigUqVKOR6IebZiHVQOHjxIdHS01WWIiIhIHuzbt48qVaqc95hiHVSypnbet28foaGhFlcjIiIiFyM+Pp7o6Givh3KeS7EOKlm3e0JDQxVUREREipmL6bahzrQiIiJSZCmoiIiISJGloCIiIiJFVrHuo3KxXC4XGRkZVpchJYyvry8Oh8PqMkRESrQSHVQMw+Dw4cPExsZaXYqUUOHh4VSoUEHz+IiIFJASHVSyQkq5cuUIDAzUHxPJN4ZhkJyczNGjRwHzacYiIpL/SmxQcblcnpASGRlpdTlSAgUEBABw9OhRypUrp9tAIiIFoMR2ps3qkxIYGGhxJVKSZf1+qQ+UiEjBKLFBJYtu90hB0u+XiEjBKvFBRURERIovBZVSonr16owdO/aij//999+x2WwaMSUiIpZSUClibDbbeV+jRo3K0+cuX76cIUOGXPTx7dq149ChQ4SFheXpfBdLgUhERM6nxI76Ka4OHTrkWf72228ZOXIkW7Zs8WwLDg72LBuGgcvlwsfnwv8Yy5Yte0l1+Pn5UaFChUt6j4iIFHOGAYYb7EVnFKNaVIqYChUqeF5hYWHYbDbP+ubNmwkJCWH27Nm0aNECp9PJX3/9xY4dO+jduzfly5cnODiYVq1aMW/ePK/PPfvWj81m49NPP6Vv374EBgYSExPDjBkzPPvPbumYOHEi4eHhzJkzh/r16xMcHEy3bt28glVmZiaPPPII4eHhREZG8vTTTzNw4ED69OmT5+tx6tQp7r77biIiIggMDKR79+5s27bNs3/Pnj306tWLiIgIgoKCaNiwIbNmzfK8d8CAAZQtW5aAgABiYmKYMGFCnmsRESnRXJkwvh181MlcLiJKVVAxDIPk9MxCfxmGka/f45lnnuH1119n06ZNNGnShMTERHr06MH8+fNZvXo13bp1o1evXuzdu/e8nzN69GhuvfVW1q1bR48ePRgwYAAnT5485/HJycm89dZbfPnll/zxxx/s3buXJ5980rP/jTfe4Ouvv2bChAksWrSI+Ph4pk+fflnfddCgQaxYsYIZM2awZMkSDMOgR48enuHAQ4cOJS0tjT/++IP169fzxhtveFqdXnjhBTZu3Mjs2bPZtGkT48ePJyoq6rLqEREpsWL3wNGNcGQ9HFxtdTUeperWT0qGiwYj5xT6eTe+1JVAv/y71C+99BLXXXedZ71MmTI0bdrUs/7yyy8zbdo0ZsyYwbBhw875OYMGDaJ///4AvPbaa7z33nssW7aMbt265Xp8RkYGH374IbVq1QJg2LBhvPTSS57948aN49lnn6Vv374AvP/++57WjbzYtm0bM2bMYNGiRbRr1w6Ar7/+mujoaKZPn84tt9zC3r176devH40bNwagZs2anvfv3buX5s2b07JlS8BsVRIRKZFcmebtmsuZMiF2T/byZ12g78fQ9LbLr+0ylaoWlZIi6w9vlsTERJ588knq169PeHg4wcHBbNq06YItKk2aNPEsBwUFERoa6pkSPjeBgYGekALmtPFZx8fFxXHkyBFat27t2e9wOGjRosUlfbczbdq0CR8fH9q0aePZFhkZSd26ddm0aRMAjzzyCK+88grt27fnxRdfZN26dZ5jH3roIaZMmUKzZs3417/+xeLFi/Nci4hIkZWRAuOugC9uvLzPObXbe33axQ/AKEilqkUlwNfBxpe6WnLe/BQUFOS1/uSTTzJ37lzeeustateuTUBAADfffDPp6enn/RxfX1+vdZvNhtvtvqTj8/u21qW677776Nq1KzNnzuTXX39lzJgxvP322wwfPpzu3buzZ88eZs2axdy5c7n22msZOnQob731lqU1i4jki2Nb4fgW8A83W0Ni90BqPPiHmuFl9yKodXXOjrHpybD0A4iMgQa9s1thzg4qkP15FipVLSo2m41AP59CfxX07KWLFi1i0KBB9O3bl8aNG1OhQgV2795doOc8W1hYGOXLl2f58uWebS6Xi1WrVuX5M+vXr09mZiZ///23Z9uJEyfYsmULDRo08GyLjo7mwQcfZOrUqTzxxBN88sknnn1ly5Zl4MCBfPXVV4wdO5aPP/44z/WIiBQZhgEftIZv74R1U7K3nzg92GD6Q/B1P1jygff7XJkwdyT89gp8PxAWjYXfX4f3W8Gi/5jHVLsq+/hdCwv0a1yMUtWiUlLFxMQwdepUevXqhc1m44UXXjhvy0hBGT58OGPGjKF27drUq1ePcePGcerUqYsKauvXryckJMSzbrPZaNq0Kb179+b+++/no48+IiQkhGeeeYbKlSvTu3dvAEaMGEH37t2pU6cOp06dYsGCBdSvXx+AkSNH0qJFCxo2bEhaWho///yzZ5+ISLF2YgdwukV79VfZ249vh8ot4J9p5vqisdD+EXM58Rj890pIPp59/LxROT+73XCw22HXH2YQanUf9Hjr8vq/XAYFlRLgnXfe4Z577qFdu3ZERUXx9NNPEx8fX+h1PP300xw+fJi7774bh8PBkCFD6Nq160U9Vbhjx45e6w6Hg8zMTCZMmMCjjz7KDTfcQHp6Oh07dmTWrFme21Aul4uhQ4eyf/9+QkND6datG++++y5gzgXz7LPPsnv3bgICAujQoQNTpkzJcW4RkUuWdAJ+HgHN74I61xfsudISYMoAiLnODBEA2+flfuy0IbDm6+z1oDPm0Fo3JTukVO8Au//M3mdzgOEyl8vUgKi6ZlAxd1oWUgBshtWdDC5DfHw8YWFhxMXFERrqfQ8tNTWVXbt2UaNGDfz9/S2qsHRzu93Ur1+fW2+9lZdfftnqcgqEfs9EiiDDKPg/rL88C0v/ay7f/g0cXAVXPmz+jNsPVww8dw0b/me2iHR8Cv7+EMrWhVrXwJ4lsHICVGllHteoHwSWgb8/gtn/MreNPGn2Ofm8G+xdcuE6A6Pgya3mJG4fd4YjG+DaF+Gqx2B0ePZxzx2BFZ+bQeaaF+Cvd2D+6VGdj2+C0Ep5uUrndL6/32dTi4rkmz179vDrr7/SqVMn0tLSeP/999m1axd33HGH1aWJSGnxz3SzpeOmTyGmS8GdJ25f9vIUc5oHdv1pzj/iSoOjm6DSFVC9PYRVMffHH4K1k7MDAMCCV82f/T4z+5W40mHdt+a2uS9C24e9z7t9PpzaZYYUuy+4M85fZ/Jx+PAqc34UAIcftBxshqguo8xbP9e9BL7+3udqegdsmAbN78z3kHKp1KIi+Wbfvn3cfvvtbNiwAcMwaNSoEa+//nqO2zoliX7PRCy2YwFkpkLt68DhA6NOP5/MPxye2XPet16Wb++CTTMufJxfMNw4DkIrw+RbIDUue59/OKTG5v4+uw+4LzA7bKv7Ibo1TL3fXA+vCnV7wrUvwI7f4PtBOT+jQW+49Qtz2e2CY1ugXP1Cv7WjFhWxRHR0NIsWLbK6DBEpDfYuhcXjYPPP5nrdHtD/m+z96UmQEmsGgYjql3cuVwYkHYfQiuYQ3vRk78nRwLyN88eb5nK1q8AZkj3T6w+Dc//cs0NK0/5w4/tm8PILglWT4KdHc76vQmOodS10ehr8As0+LDa72VKSpX4vqNrWux9K1jmy2B1QvgFFnYKKiIgUXUknwBkMPs7sbZnp8M3tkHIqe9uWWXAoe8JH/ILgy75mn4yHl0JkLbPvyuqvYNNP0G4YVG1ntsKcz8md8PWt5s/ub8C80ZCekPO4a56HOt0hbi806GO2UGQNBV56xhDh4asgI9m8HXOmKq2g13tmPY7TD59tMQg2TM0eIhxQBto/ar7ObAFpdW/utV81AvzDoP0IWPGZ2ZpTuwBvhxUQBRURESlc6cnwy9NQvhG0eSB7e+xe2Ps3BJc1WzCCysLkW6HqlXD3j9nH7f4zO6RUaW32u9jzF/x2Rqf91FizYyuY/Toia5mjYWacfqzItjlmK8Qd35t9WfYthyXjzJoa3wwbfzRvJ/30aPbcJLOyn23mUbMzXP386VpamK8sDh+4brR3UImsBW43lKlphh+ABxdBVAz4+OX8/Eb9soPKU9sv7anGtbtkB5PoVhf/viJGQUVERHIXtx/2r/CevTQ/TBtitmoAtLwHHKdnvZ7+cM5bFQA7f4fkk+YImD/fzu6M2mIw9BoL638wg8q2X3M/36pJUP2q7FszWQy3OSmaw8/sxApmQMnq4Jo1x4jDz+z/cWJ7zs8+M0DlxuELXV+DOf82O62COUfJ4F/M7xEeDRUanfv9ze6AQ2vNkUGXElJKEAUVERHJ3aRe5v/19/kQmvW/8PEX4+im7JACcHyb2U8i+WTuISXLWzFQrZ05siZL41vMn/V7QVA5SDrHs8qObIDxbc3lwEhofCv8PT57v+v8jxvhyofN+Us+vdZsDer3KUy+Dep2P//7znx/7esgsnb2tpDy0OeDc78ni8MXbnjn4s5TQimoiIhITmmJ2bcm1n1rBpXUOHMkSY2O5jwcebFykvf6kQ0QFAWfnTFpWlBZSDpmtmRUbgl7F5ujV7ImIKvSGrq9nn2bxccJnZ+BmY9D+cZmy8OhNeZtnCMbvM/Xdhi0e8QMGau/hPXfm9vL1jNHv2TN6HrrF2Yn3Ni9UKebGRgeWmK2wjiD4bF/zA6zF8Nmg7J1LuUqyRkUVEREJKc9Zzxt/NgWc2TJkv+aw153/AZXDs29T8X5/PpCdktGSEVIOASH15l9ME7tMrdf9Zg5v8fRzYBhBoXJiyEgwgwndh+of2POc7e8B8KrmeFl8TgzqHR6GhKPwLrvYP8yszNqq/vMviM1O5mjYrqMhrDKZkfbf6ZlB5XaXcwOuRWbZp/DLzB7OSjy0r675JmCSgnVuXNnmjVrxtixYwGoXr06I0aMYMSIEed8j81mY9q0afTp0+eyzp1fnyMiFtr5e/ZywkEYE22GiyyrJpnrexaZ4cE3AK563Jw4DE4Hmw/Mqdkb9oXjW2H5Z+a+6CvNDquznoRVX2YP0y1bH9qeniK+XL3T2+rBbV+Zk6eFVT53vTZb9gRvnZ81g0vWRGut7zc7ywZEeD8J2Mcv+zNtNjMAtR4CFZuZIUWKBAWVIqZXr15kZGTwyy+/5Nj3559/0rFjR9auXUuTJk0u6XOXL19OUFD+/os3atQopk+fzpo1a7y2Hzp0iIiIiHw919kmTpzIiBEjiI2NLdDziBRbKbGQkWLO/ZGb800zbxje/UjMjWZgyZLbCBgwh9mGVDA7imZ1bl3wivcxg36G+APg458dUkKrwEOLzY6mZ7LZzD4ol8Lhmx1SslzMqBeHD/R488LHSaGyX/gQKUz33nsvc+fOZf/+/Tn2TZgwgZYtW15ySAEoW7YsgYGBFz4wH1SoUAGn03nhA0WkYLgy4fOuMK6FOUHZ2RaPg/+rAUs/zP39+5aZ84H4BcO/dsEN72bvc5x1y8X3jP+uLHwDvr7ZnCPkXCNwytY3g0REdbhnjjm81+bIfmKvyFn0W1HE3HDDDZQtW5aJEyd6bU9MTOT777/n3nvv5cSJE/Tv35/KlSsTGBhI48aN+eabb3L/wNOqV6/uuQ0EsG3bNjp27Ii/vz8NGjRg7ty5Od7z9NNPU6dOHQIDA6lZsyYvvPACGRnmcyUmTpzI6NGjWbt2LTabDZvN5qnZZrMxffp0z+esX7+ea665hoCAACIjIxkyZAiJiYme/YMGDaJPnz689dZbVKxYkcjISIYOHeo5V17s3buX3r17ExwcTGhoKLfeeitHjhzx7F+7di1XX301ISEhhIaG0qJFC1asWAGYzyzq1asXERERBAUF0bBhQ2bNmpXnWkQKjdsNm36GjzrAsc2QkWQ+aC6r/8WpPeYxf75tzkPyy9Mw80lzeG8Ww8juYFrvBnNIcIvBZt+OK+6GJ7ZAk9vh5s/hxVh47hCMPGXezjnbFXdD97NaKMrWzV6u1Mwc3jvyBFz5YH5fDSkhStetH8MwZwQsbL6BFz0HgY+PD3fffTcTJ07kueeew3b6fd9//z0ul4v+/fuTmJhIixYtePrppwkNDWXmzJncdddd1KpVi9atW1/wHG63m5tuuony5cvz999/ExcXl2vflZCQECZOnEilSpVYv349999/PyEhIfzrX//itttuY8OGDfzyyy/Mm2c+bjwsLCzHZyQlJdG1a1fatm3L8uXLOXr0KPfddx/Dhg3zCmMLFiygYsWKLFiwgO3bt3PbbbfRrFkz7r///ou6bmd/v6yQsnDhQjIzMxk6dCi33XYbv//+OwADBgygefPmjB8/HofDwZo1a/D1NedyGDp0KOnp6fzxxx8EBQWxceNGgoODL7kOkUI370VY/J73tjWTzT4XPww2h+b2Ge89o+vyT8xXZC1zxtcvbjSncAezHwmY//3q+Xb2e276yPscdjsMnmWOCpr+MGydbW6PvhKaD4Amt8Ab1c1tAbncFi7k58xI8VK6gkpGMrxmwVMg/33wkjpm3XPPPbz55pssXLiQzp07A+Ztn379+hEWFkZYWBhPPpl9f3j48OHMmTOH77777qKCyrx589i8eTNz5syhUiXzerz22mt07+49J8Dzzz/vWa5evTpPPvkkU6ZM4V//+hcBAQEEBwfj4+NDhQoVznmuyZMnk5qayhdffOHpI/P+++/Tq1cv3njjDcqXLw9AREQE77//Pg6Hg3r16tGzZ0/mz5+fp6Ayf/581q9fz65du4iOjgbgiy++oGHDhixfvpxWrVqxd+9ennrqKerVMzvsxcTEeN6/d+9e+vXrR+PGjQGoWbPmJdcgkm/cbnNUTNW2ZkfVwxvg5A5zErbMNNg4w7yVElQWln1ivqdSc7OPxs6F5jDfrGfNJJ8wZ3oFqNDEHHGTZemHcGBFdkgB87bMxbI7zNaXyi2yg0rV060sZ4aT6Av/N0rkTKUrqBQT9erVo127dnz++ed07tyZ7du38+eff/LSS+ashi6Xi9dee43vvvuOAwcOkJ6eTlpa2kX3Qdm0aRPR0dGekALQtm3bHMd9++23vPfee+zYsYPExEQyMzMv+JTL3M7VtGlTr4687du3x+12s2XLFk9QadiwIQ5H9qyLFStWZP369Zd0rjPPGR0d7QkpAA0aNCA8PJxNmzbRqlUrHn/8ce677z6+/PJLunTpwi233EKtWrUAeOSRR3jooYf49ddf6dKlC/369ctTvyCRS2YYMOUOc/r4QT+b84Ms/wRm/8u8/RJzPXw/0Jyg7KZP4a93zIfenaliU7h/gdlKseQDc0bU3HR4ArbMhnVTzPWsn1naPZI9Y+ylqN4+e7nMGSH/4b9h7xLvh+KJXITSFVR8A83WDSvOe4nuvfdehg8fzgcffMCECROoVasWnTp1AuDNN9/kP//5D2PHjqVx48YEBQUxYsQI0tMvMLviJViyZAkDBgxg9OjRdO3albCwMKZMmcLbb7994TfnQdZtlyw2mw23210g5wJzxNIdd9zBzJkzmT17Ni+++CJTpkyhb9++3HfffXTt2pWZM2fy66+/MmbMGN5++22GDx9eYPWIAGaLx5bT/aH+/gia3Aa/nm7ZXDkB/pmaPYvq1PvMnz7+EFrJnBDNxx+uezn7VsoVA2HZx2aH2sBI8/Oz1O4CDftA3w/NPi2HT/+PQednoeFNUKZG3r5DtXZw8wSzs+yZt3TK1csecixyCUpXZ1qbzbwFU9ivPNx/vfXWW7Hb7UyePJkvvviCe+65x9NfZdGiRfTu3Zs777yTpk2bUrNmTbZu3XrRn12/fn327dvHoUOHPNuWLl3qdczixYupVq0azz33HC1btiQmJoY9e7wfa+7n54fL5brgudauXUtSUpJn26JFi7Db7dStW/c878y7rO+3b98+z7aNGzcSGxtLgwbZjzSvU6cOjz32GL/++is33XQTEyZM8OyLjo7mwQcfZOrUqTzxxBN88sknBVKriJfj27KX574Ab9fxnt49Nc77eP8weOBPeGQ1jFgPw5abE5llcQabQ35veNd8+F6ta8ztkTHmPjD/+9Tynuz3NBtgzqKal9aULI1ugspX5P39ImcoXS0qxUhwcDC33XYbzz77LPHx8QwaNMizLyYmhh9++IHFixcTERHBO++8w5EjR7z+CJ9Ply5dqFOnDgMHDuTNN98kPj6e5557zuuYmJgY9u7dy5QpU2jVqhUzZ85k2rRpXsdUr16dXbt2sWbNGqpUqUJISEiOYckDBgzgxRdfZODAgYwaNYpjx44xfPhw7rrrLs9tn7xyuVw55nBxOp106dKFxo0bM2DAAMaOHUtmZiYPP/wwnTp1omXLlqSkpPDUU09x8803U6NGDfbv38/y5cvp168fACNGjKB79+7UqVOHU6dOsWDBAurXr39ZtYp4STllztJa7wao2w0Sj8G+pebzbi6kzUPm3CjHt0LHp8yWi/PxC8oOIn3Gm0OI2w7zPqZpf/MZOuXqmw/JEylCFFSKsHvvvZfPPvuMHj16ePUnef7559m5cyddu3YlMDCQIUOG0KdPH+Li4s7zadnsdjvTpk3j3nvvpXXr1lSvXp333nuPbt26eY658cYbeeyxxxg2bBhpaWn07NmTF154gVGjRnmO6devH1OnTuXqq68mNjaWCRMmeAUqgMDAQObMmcOjjz5Kq1atCAwMpF+/frzzzuU/ZCsxMZHmzZt7batVqxbbt2/nxx9/ZPjw4XTs2BG73U63bt0YN24cAA6HgxMnTnD33Xdz5MgRoqKiuOmmmxg9ejRgBqChQ4eyf/9+QkND6datG++++26O84tckoOr4dhWKN/QDAubZpjPmnnhBHx3t/k8G2fOkXOEVYXur5v9V3b8Bp3+ZXZazYuQCt5zomTxDYBbJuTcLlIE2AzDMKwuIq/i4+MJCwsjLi4uRyfP1NRUdu3aRY0aNfD397eoQinp9HtWws18Eg6thTv/Z7aCBISDM9Rs+Vg72ezg6jxj6HpmGvz8GFTv4P204WNb4aOOkJmS8xy3fgnf3eW9rcVgs9Ujqvb5Z5AVKabO9/f7bGpRERHJTcopc8QNwOtn3A7xDTInUgM4ugn6/Dd735ZZsOZr87XxR/OJvpG14H/3ZIeUwChwZ2T3Nzk7pIA59DiqtrmskCKlnIKKiAjAhqnmSBtffzDc5lTvucnI7hjOmq+9g8rRTdnLW2dnzycC5qibhxabt18Ajm+H/7YxR+ucyTfQfF6OiAAKKiJSGq2dYs7pUbeHOVGZw5k9KVqWXX94r5dvBEc25Pys1DjzdpDNZk7GlhubA3r/NzukgNli0uZBWPI+BJeH5nfCsS3Q9TXv20kipZyCioiUDvuWmXOSXP1vmPaAuW3lRPNn2OlbO75B5twix7fC/uXmtsotIboNXP+y+aC9H4dB8vHsz329qnk7p0Ij2Pm7uW3QTDPYbJphTiMfGAlBkTlruu4lqNPNfOaNMyT/v7NICVDig0ox7issxYB+v4qYTT+bs612e82cY2T3Ilh6+tbM5p/Nn1/0zvm+uNNz7vT5rxlUAPYsNltIWt2X/VTfut3hvnlm4HFlwLY55vbk49khBcyQEhBuPpTvfOwOqNEhD19UpPQosUEla6bT5ORkAgICLK5GSqrkZPMhl2fPrCsW+XaA+XPNV+AXDOmJ5z/+bDU6Zi9Xa2e+zlamBtz+tbkcdwCO/HP6+TmnQ2vllmZIEZF8UWKDisPhIDw8nKNHjwLmfB429Z6XfGIYBsnJyRw9epTw8HCv5xRJPnJlmLdgotuYrQ9LP4QtM83hvx2ezG7pAHNo8JmyQkrM9eYtm7PFXA8Jh8wp5yffZj6j5lLnJwmrbL4Gz4KUWKjWNk+PzBCRcyuxQQXwPNU3K6yI5Lfw8PDzPj1azsEw4MR286F19vOEvCUfwLwX4cqHofX9MOdZc0TOrj/Mz+j8tHncoXUw8/Gc779iIPR8xww7Pw41hxynnITub0KbIdnHjVgP/pf2wE0vubW8iEi+KLETvp3J5XKRkZFRiJVJaeDr66uWlLxaMxmmPwSdnoGrnzW3HVoHi8eZU8Jf8xwc3QwfXmXOOQJmqDm50/tzal1rPnDv5I7sbSEV4ZZJULVNzvOmJ5tPG67cQvOTiFhIE76dxeFw6A+KiNXSEsxhun6BZkgBWPi6OSlaWjxMuiF7ErTQSvDzCO/3Z4WUQTPNOU9WfAY75uc8T9W2uYcUMM9dpWW+fB0RKRylIqiIiMXmjYbF74HdF4LLee/7612YP9p729khpdkAsz9J6weg+lVQpbU5nDf5hDnzq9sFv71sHquH6omUKKXi1o+IFLL0ZNi/zLwNY3PA+y0u7n2VW8CBld7bGt8C/T698Ht3/wUrJkDPtyAg4tJrFpFCo1s/IlK4Uk7BjgVQoQn4OOHrm+HYZu9jqrY1Jzg7sd0MFZlpsOGH7P1tHoJ2w+E/TbKnlW89xBzdczGqX2W+RKREUVARkcuzbzl81Q/S4iCihjkd/LHN4AwzO8JmmHPN0OQ2iG5tvprdYT6BOCuohEVD99fN5WrtYddCc7nra+DQHDUipZn9woeIiJwlM80cbTP1AfisixlSAE7tgn1LzcnWHvwDntpuzldSppb5ROAzBZaBmleby+0fzd7e8x3wDzdv+SikiJR66qMiIpfu+0Hwz7Ts9RodIT0pu3/JdS9D+0ey9xtG7sOBU06Zt4Hq9sw5eZvDT0OIRUoo9VERkfwTfwim3AF+QVCpOdTs7B1SwqvBwJ/g99ezg0qLQd6fca7AERAB9Xvl3O7jzI/KRaQEUFAREW/HtsCyj80Orde9ZA77PbjK3Lf7T3OY8ZmqtTd/tnnAnLStYZ/Lm+VVROQMCioiAqnxsOBVOLHDfGpwRpK5ff3/ID3BXG50M2z9JeeD/iJrmT8DIqD/5MKrWURKBUs707pcLl544QVq1KhBQEAAtWrV4uWXX6YYd5sRKfoMI+cD/OY8C39/CNvnZocUyA4pMV3h5s9g8Gyzk+u988zbOyEVofmdhVa6iJQ+lraovPHGG4wfP55JkybRsGFDVqxYweDBgwkLC+ORRx658AeIyKX7+mbzFs39v5mzuMYdgLXfeh9TpqY5r8nsp8z1VveZPys2yZ58LbpV4dUsIqWWpUFl8eLF9O7dm549ewJQvXp1vvnmG5YtW2ZlWSIlV1oCbJ9nLs9+Gm7+HL4faM53UrUd7F1s7qt9HVxxt9lp1tcfal9rXc0iUqpZGlTatWvHxx9/zNatW6lTpw5r167lr7/+4p133rGyLJGS6/CG7OUts8yHAu5fbs5b0us/5uRsayabTy/29Yd7ZltWqogIWBxUnnnmGeLj46lXrx4OhwOXy8Wrr77KgAEDcj0+LS2NtLTse+vx8fGFVapIyXB43RkrhvlAQICr/w1l65jLlZoVdlUiIudkaWfa7777jq+//prJkyezatUqJk2axFtvvcWkSZNyPX7MmDGEhYV5XtHRekqqiBfDMF9Zts+DxKPZ64dOB5WQStnbnKHmlPYiIkWQpTPTRkdH88wzzzB06FDPtldeeYWvvvqKzZs35zg+txaV6OhozUwrpcvO32HvUoi5znzaMJjDinf+bvYpSTwK/b+Bo5vg2wFQsSk88AdkpMC4FhB/APp8CL+PMbf1+a/5WSIihaTYzEybnJyM3e7dqONwOHC73bke73Q6cTo1Y6WUUtMegp0LIOGQub7qSxixzryd80VvSI3LPvarfuaIHoBDa81p6he8ZoaU0Crmc3ca9TOfpaNp6kWkCLM0qPTq1YtXX32VqlWr0rBhQ1avXs0777zDPffcY2VZIkXPsS2w9qzJ1OL3my0of7zpHVLAfDjgqV3Z6xN7nl6wQbcx4BdYoOWKiOQXS2/9JCQk8MILLzBt2jSOHj1KpUqV6N+/PyNHjsTPz++C79dDCaVE2/KLORHbTZ/Czt/gt1ey90XU8A4iARHw4F/gdsHWOdnzn5yp2lXQ9RXzeT0iIha6lL/fenqySFE1Ksz8GVIJfAPg5A5zPTIG7ppmPsH4wApzW6/3oMVAczkjFf53L2yZbT53p0priKgGMdeD3VHY30JEJIdi00dFRM4h+WT2csJB82dQWRiyEALLmMHl/vnmcTab2aKSxdcfbv/abF1RMBGRYk5BRaSoMAyzz0lEdViVyxD9/t9CWGXvbYFlzv15CikiUgIoqIgUtvRkc1bY8o2gXL3s7VvnwA+Dc39PRA2o0qJw6hMRKUIUVEQK21/vwh//Zy73+RB8/MAv2By9c6Yeb4FvICwdn/0gQBGRUkZBRaSwpMbD3Bdg5cTsbdMfzHlczc7QtD80vd1cb577IyVEREoDBRWRwuDKhL/e8Q4puWnzIHR/o1BKEhEpDhRURArKjgXmEOGoGPjz7ewZZbPc9Kk5iVuHJ2Dy7eDOhHbDralVRKSIUlARKQiZ6WbH2JRTue9vOwya3GK+AB5YCIYbwqoUXo0iIsWAgopIfko4Aqu/hNDKuYeUHm9B1bYQWdt7e2StwqlPRKSYUVARyU9zX4B132avl28ERzaYyw8vhXL1ralLRKSYUlARuRxuF2ADux3Sk7xDCkCf8eY2uw+UrZfrR4iIyLkpqIjkVdwB+KI3+IdC9/+DH4dl76vcAq5+Dio2MV8iIpInCioieWEY8MM9cGKbuf7ptdn7rn8V2g3L/X0iInJJFFRELlVaAhxcDfuWem+vfR1c9xKUb2BNXSIiJZCCisjFykiFX5+HFZ+D4TK31e4CPd8Bmx3Co62tT0SkBFJQEbmQHQvM2zypseZcJ2eq0w0iqllSlohIaWC3ugCRIufUHji0zlze8Rv8715IOWmGlJCKcOsXZmdZ/zCof6O1tYqIlHBqUZHSzTDg2BZzAjaHjzlJ2yfXQPJxczjxsc3mcWFVofc4iG4DvgFQt4c55b1vgLX1i4iUcAoqUrqt/x6m3m8Gkfo3QGqcGVLADCl2H2h1P3R+BgLCs9/n8DVfIiJSoBRUpHTb+KP5M24vLP1vzv093oSW9xRuTSIi4qGgIqWT2wUrJ8Dmn831oHKQdNRcrnYVlK1jPljwioHW1SgiIgoqUkqt+BxmPZm9/vgmWPoBrPkGeo2FqBjLShMRkWwa9SMl16nd8N3dsH8lJB03R/MAJJ+EP97MPq5hX7MjbftHYehShRQRkSJELSpScv3xptkHJasfCkCFJmaH2cQjEFED+n4I5RtaV6OIiJyXgoqUXEf+ybnt8On5UZxhcPvXCikiIkWcgoqUTG43nNhhLkdUN6e5D6kIB1ZAYCRUaQ3BZS0tUURELkxBRUqm2N2QFg8OJwxbkT3niR4YKCJSrCioSMniyjBH8/wz3Vwv31ATs4mIFGMKKlKybP4ZVk40l212aHaHpeWIiMjlUVCRkmX9D+bPcg3gjm8hvKq19YiIyGXRPCpScqTGwbZfzeV+nyqkiIiUAAoqUnLs+hNc6VCmloYdi4iUEAoqUnLsXGD+rHW1tXWIiEi+UVCRksGVCdvnm8s1FVREREoKdaaV4m/mE7D8U3PZNwhqdLC2HhERyTdqUZHi7fh2WP5Z9vpNH4F/mHX1iIhIvlKLihRfp/bAtwMAA8rUhP5ToGxdq6sSEZF8pBYVKb5+HgHHNoOPP9z8uUKKiEgJpBYVKT4yUmH1l2brybKPYcdvgA0Gz4ZKza2uTkRECoCCihQPrgyY1Av2L/PeXqcbVL7CmppERKTA6daPFA87fssZUso3gm5jrKlHREQKhVpUpHjYOMP8GVwBEg9D9Q4w8Cew2aytS0RECpSCihR9aYmwZaa53O9T8AuCyFoKKSIipYCCihRtGakw6ylIOWU+ZLBqW3Do11ZEpLRQHxUpeo5tgWkPwpF/4Ms+sHayub37mwopIiKljP6rL0XPL8/Cjvmw9htz3eGEnm9B3W7W1iUiIoVOQUWs58qAH4dBUBQcWgu7//Te3+NNuOJua2oTERFLKaiI9Xb9AeumeG8Li4bW90PFZlCzkyVliYiI9RRUxHoHV3mvB5WD/t9AhcbW1CMiIkWGgopYb/+K7OWy9eChJWBXP28REVFQESvF7oWZT8K2Oeb6Nc9D0/4KKSIi4qGgItbYPBOmPwypseZ6QAS0ewR8nJaWJSIiRYuCihS+xe/Dr8+ZyxWbQcvBULWdQoqIiOSgoCKF6+AamPeiuXzlw9BlNPj4WVqSiIgUXQoqUngMA355BtyZ0KAPdH1Nz+sREZHzUq9FKTz/TIO9S8DHH7q9rpAiIiIXpKAihePgGpj2gLnc5kEIrWhpOSIiUjwoqEjhWPUFuNKh1rXmMGQREZGLoKAiBc8wYMtsc7nNg+DwtbYeEREpNhRUpODtWQwJB8E3CGp0tLoaEREpRhRUpGClJcCMYeZywz7g629pOSIiUrxYHlQOHDjAnXfeSWRkJAEBATRu3JgVK1Zc+I1SPMx6Ck7uhNAq0PVVq6sREZFixtJ5VE6dOkX79u25+uqrmT17NmXLlmXbtm1ERERYWZbkhyP/wLa5sPYbsNmh36fmNPkiIiKXwNKg8sYbbxAdHc2ECRM822rUqGFhRZIv3C74si8kHjHX698I1dpaW5OIiBRLlt76mTFjBi1btuSWW26hXLlyNG/enE8++eScx6elpREfH+/1kiLo0NrskAJwxV3W1SIiIsWapUFl586djB8/npiYGObMmcNDDz3EI488wqRJk3I9fsyYMYSFhXle0dHRhVyxXJQd87OXG94ENa+2rhYRESnWbIZhGFad3M/Pj5YtW7J48WLPtkceeYTly5ezZMmSHMenpaWRlpbmWY+Pjyc6Opq4uDhCQ0MLpWY5D7cb5o2ExePM9e5vQpsh1tYkIiJFTnx8PGFhYRf199vSFpWKFSvSoEEDr23169dn7969uR7vdDoJDQ31ekkRMveF7JASVA7q97K2HhERKfYs7Uzbvn17tmzZ4rVt69atVKtWzaKKJM+2zYMl75vLN4yFK+4Gu8PSkkREpPiztEXlscceY+nSpbz22mts376dyZMn8/HHHzN06FAry5K8WHK6JaX1EGg5WCFFRETyhaVBpVWrVkybNo1vvvmGRo0a8fLLLzN27FgGDBhgZVlyPmmJkHwSVk6Elac7Pcfth50LzeW2wywrTURESh5Lb/0A3HDDDdxwww1WlyEXY/mnMOd5yEzJ3hZV5/QtHwOqXQURum0nIiL5x/KgIsWEYcBfY71DCsCEbuZPhx90ebHQyxIRkZLN8mf9SDFxahfE7ct9X2RtuPVLiG5duDWJiEiJpxYVOb+4AzD7X7D5Z3M9+kqw+0DCQbjhXfALhsotwGaztk4RESmRFFRysWzXSSYu3kVMuRAeu66O1eVYa9Wk7JACUKMjXP1vBRMRESkUCiq5OBKfyqz1h7myZrrVpVjv2Ol5bpyhUL0DtBiokCIiIoVGQSUXTh+z605aptviSoqA41vNnzd/DjHXWVuLiIiUOgoquXD6mpOVpWWU4qBiGHBgJRzdaK5HxVhbj4iIlEoKKrnwP92ikprpsrgSi+xbDj8/BkfWm+s2O4TpSdUiIlL4FFRyUapbVFJOwVc3QVp89jbDrSnxRUTEEppHJReluo/K/pVmSPELNocf2xyaFl9ERCyjFpVc+Ge1qJTGWz8HVpg/6/aAlvdA/d4QEGFtTSIiUmopqOTC06JSGm/9HFhp/qzS0vwZFGldLSIiUurp1k8usoJKusuN221YXE0hcmXA/tMtKpVbWluLiIgICiq5yrr1A2ZYKRVWfwUvR0HKSQgqCxUaW12RiIiIgkpuslpUAFIzSng/FVcGrJwEPw7N3tZuOPj4WVeTiIjIaeqjkgsfhx2H3YbLbZT8kT+/vQKLxmavN+gNre6zrBwREZEzKaicg7+PnaR0V8nuUOt2w7pvzeXgCnD/fAirYm1NIiIiZ1BQOQenr8MMKiV1iLLbBfNHQ8Ih84GDI9aBj9PqqkRERLyoj8o5ZPVTSS2pLSorJ8Ci/5jLdbsrpIiISJGkoHIOJXrSN8OAFRPM5ai6cM0L1tYjIiJyDrr1cw4lehr9/SvgyAZwOOGeXyCwjNUViYiI5EotKueQfeunhLWouN0w93QLSuObFVJERKRIU1A5B88TlEtSi4rbBd/fDXuXgG8gXP2c1RWJiIicl4LKOWTf+ilBLSrb5sKmn8xbPjeOg7DKVlckIiJyXgoq5+D0MVtUStSon+WfmD/bDDFv+4iIiBRxeQoq+/btY//+/Z71ZcuWMWLECD7++ON8K8xq/r5ZT1AuIS0qJ3fC9nnmcst7rK1FRETkIuUpqNxxxx0sWLAAgMOHD3PdddexbNkynnvuOV566aV8LdAqWS0qJaaPyorPzZ+1u0CZmtbWIiIicpHyFFQ2bNhA69atAfjuu+9o1KgRixcv5uuvv2bixIn5WZ9lnL4laHhyRor5dGTQc3xERKRYyVNQycjIwOk0ZzKdN28eN954IwD16tXj0KFD+VedVeIP0jhxEW1sm0rG8OQNUyHlFIRVhZjrra5GRETkouUpqDRs2JAPP/yQP//8k7lz59KtWzcADh48SGRkZL4WaImdC+m/42mG+kwv/i0qySfhz7fN5ZaDwe6wth4REZFLkKeg8sYbb/DRRx/RuXNn+vfvT9OmTQGYMWOG55ZQsXZ6ErRwW2LxHp7sdsMP98DJHRBSCVoMsroiERGRS5KnKfQ7d+7M8ePHiY+PJyIiwrN9yJAhBAYG5ltxlgkwv1M4icV7ePKyj2HnAnNytzt/0Cy0IiJS7OSpRSUlJYW0tDRPSNmzZw9jx45ly5YtlCtXLl8LtERAdotKcnqmxcXkUUZq9i2f61+G8g2trUdERCQP8hRUevfuzRdffAFAbGwsbdq04e2336ZPnz6MHz8+Xwu0xOkWlVBbCknJqRYXk0frvoWkoxBaBa4YaHU1IiIieZKnoLJq1So6dOgAwA8//ED58uXZs2cPX3zxBe+9916+FmiJgHAMbAC4kk9ZXEweuN2weJy5fOVD4PC1th4REZE8ylNQSU5OJiQkBIBff/2Vm266CbvdzpVXXsmePXvytUBL2B24/EIBsKWctLiYSxS7Fz7vCie2gTMMWqg1RUREiq88BZXatWszffp09u3bx5w5c7j+enNujqNHjxIaGpqvBVrF7R8OgD011tI6Lsnh9fDpdbB/mbne+Wlwhlhbk4iIyGXIU1AZOXIkTz75JNWrV6d169a0bdsWMFtXmjdvnq8FWsV2eoSMX0YsbrdhcTUX4chGmNATEg9DuYbwwJ/QdqjVVYmIiFyWPA1Pvvnmm7nqqqs4dOiQZw4VgGuvvZa+ffvmW3FWsmfNpUICiemZhPoX0X4ebrc5DHnh65AWB9Ft4I7vICDc6spEREQuW56CCkCFChWoUKGC5ynKVapUKRmTvZ3mCDJn2A0jibjkjKIbVGYMgzVfm8tRdaH/FIUUEREpMfJ068ftdvPSSy8RFhZGtWrVqFatGuHh4bz88su43cV4grQznW5RibAlEJeSYXEx57B1jhlSbA7o/iY8+JcmdRMRkRIlTy0qzz33HJ999hmvv/467du3B+Cvv/5i1KhRpKam8uqrr+ZrkZbwzE6bRHxRDCorJ8Ksp8zlNg9AmyGWliMiIlIQ8hRUJk2axKeffup5ajJAkyZNqFy5Mg8//HDJCCrB5gy7FWwniU8tYkFl44/w06Pmcq1roPOz1tYjIiJSQPJ06+fkyZPUq1cvx/Z69epx8mQxm3fkXCJjAKhpO1i0bv0c3wYzHjGXWw+BO6eCf8kYEi4iInK2PAWVpk2b8v777+fY/v7779OkSZPLLqpIiDKDSlXbURKSkiwu5rT9K+HDDpAaC5WaQ9fXwGazuioREZECk6dbP//3f/9Hz549mTdvnmcOlSVLlrBv3z5mzZqVrwVaJrg8qfYg/N1J2E/tAupbW49hwJx/Q2aKGVJu/VJT44uISImXpxaVTp06sXXrVvr27UtsbCyxsbHcdNNN/PPPP3z55Zf5XaM1bDbiAqsB4HNyh8XFAEs+gH1Lwccfbv8GwqOtrkhERKTA5XkelUqVKuXoNLt27Vo+++wzPv7448surChICasJiRsJiN9uXRGGAbP/ZU7qBmbH2dCK1tUjIiJSiPLUolJauMs1AqBq0gbritgy+3RIscE1z0P7R62rRUREpJApqJyHT61OADTK3AAuC0b+JJ+EuSPN5atGQMen1HlWRERKFQWV84io1YJTRjBBpJK6Z3nhnjw9Gb7sCye2QVBZtaSIiEipdEl9VG666abz7o+Njb2cWoqckAAnc2hIV/4mefN8/Gu2K7yTz3wCDq2BwEgY+JNnplwREZHS5JKCSlhY2AX333333ZdVUFGz0b85XdP+xrH7j8I76c6FsHYy2Oxw6xdQzuKh0SIiIha5pKAyYcKEgqqjyNoX3hqOfEjIsVWQngR+QQV7wsw0szUFoNV9UP2qgj2fiIhIEaY+Khdgj6zFfiMKu5EJO38vuBNlpMJvr8KYKqf7pZSDq58ruPOJiIgUAwoqF1AtMohZrjbmyu9jwO3O3xMYBqz+Cj5sD3/8H7jSzUndbngXAsLz91wiIiLFjILKBVSLCmJ8Zi+SbYFweD1snZ2/J9g2F34cCie2g8MJfT+CJ7ZA/Rvy9zwiIiLFkILKBVSPDOQUofxgu97csHJS/p5gzVfZy/0nQ9Pb1ZIiIiJymoLKBVQrY3ae/Tylg7lh+1yI3Zc/H35glTnzLMCDi6B2l/z5XBERkRJCQeUCwgJ9iQj0ZbdRkaRK7cBwm31KLtep3eaEbq50qHk1VGh0+Z8pIiJSwhSZoPL6669js9kYMWKE1aXkUC3SbFXZVqWfuWHVF+Yonbw6thW+6Q+psVDpCrithDxxWkREJJ8ViaCyfPlyPvroI5o0aWJ1KbmqXS4YgD99roSQipBwEH57+dI/yJUBs5+G/7aBoxshMApunQTOkHyuWEREpGSwPKgkJiYyYMAAPvnkEyIiiuY08fUqmEFi49E0c9gwwJL34X/3w+ELPFk5di8knYBln8Cn18LfH5q3j+p0gwf+gPCqBVy9iIhI8WV5UBk6dCg9e/akS5ei25G0fsVQADYfToC63aHT0+aO9d/BZ9fB1jk533R0E3xzB4xtDG/WhFlPwqG14BcC/afAHd9CWOVC/BYiIiLFzyVNoZ/fpkyZwqpVq1i+/OKeTJyWlkZaWppnPT4+vqBK81L3dIvK7hNJJKdnEtj5WajUHBb9B/Yugcm3gk+A2SE2qo6579fnIfOMfiw2OzS7A9o9CmXrFErdIiIixZ1lQWXfvn08+uijzJ07F39//4t6z5gxYxg9enQBV5ZTVLCTqGAnxxPT2HokkWbR4WbLSu0u8NMIcy6UzBTYv9x8rfnafGO19nDtSNj9p3mrp0LjQq9dRESkOLMZhmFYceLp06fTt29fHA6HZ5vL5cJms2G320lLS/PaB7m3qERHRxMXF0doaGiB1nvXZ3/z57bjvNGvMbe1OqtfSexeSI2HY5th44+wZxGUqQUDvtfkbSIiImeJj48nLCzsov5+W9aicu2117J+/XqvbYMHD6ZevXo8/fTTOUIKgNPpxOl0FlaJXuqWD+HPbcfZdCgh586sDrEVGkHjmwu3MBERkRLMsqASEhJCo0bek5wFBQURGRmZY3tRUM/TobZw+sWIiIhIERj1U1xkDVHecjgBi+6WiYiIlDqWjvo52++//251CedUu1wwdhucSs7gSHwaFcIurgOwiIiI5J1aVC6Sv6+DOuXNVpU1+2KtLUZERKSUUFC5BM2rhgOwet8pawsREREpJRRULkHzquYU/6v3xFpbiIiISCmhoHIJrjgdVNYdiCXD5ba4GhERkZJPQeUS1IwKItTfh9QMN5tzm09FRERE8pWCyiWw222e2z+r9qqfioiISEFTULlEWR1qFVREREQKnoLKJbpCLSoiIiKFRkHlEjWrGo7dBvtOpnAwNsXqckREREo0BZVLFOrvS+Mq4QAs3nHC2mJERERKOAWVPGhXKxKAxTuOW1yJiIhIyaagkgfta0UBsHj7CT2gUEREpAApqORBi2oR+DnsHI5PZdfxJKvLERERKbEUVPIgwM/BFdXCAfVTERERKUgKKnnULuv2j/qpiIiIFBgFlTy6KsYMKn9uO67n/oiIiBQQBZU8alolnMggPxJSM1m++6TV5YiIiJRICip55LDb6Fy3HAC/bTpqcTUiIiIlk4LKZbi2/umgsllBRUREpCAoqFyGDjFR+Dps7DyexM5jiVaXIyIiUuIoqFyGEH9f2tQwZ6lVq4qIiEj+U1C5TNfUM2//zNt0xOJKRERESh4FlcuU1U9l+e5TxKVkWFyNiIhIyaKgcpmqRQZRu1wwLrfBwq3HrC5HRESkRFFQyQdd6pcHYNa6QxZXIiIiUrIoqOSDG5tWAuC3LUeJT9XtHxERkfyioJIP6lcMIaZcMOmZbuZsOGx1OSIiIiWGgko+sNlsnlaVGWsPWlyNiIhIyaGgkk9ubGYGlUXbj3M0IdXiakREREoGBZV8Ui0yiGbR4bgNmKlOtSIiIvlCQSUf9W6m2z8iIiL5SUElH/VsUhG7DVbvjWXviWSryxERESn2FFTyUbkQf9rVigJgxtoDFlcjIiJS/Cmo5LOsTrU/rjmIYRgWVyMiIlK8Kajks26NKuDnY2fb0UTWH4izuhwREZFiTUEln4X6+9KtYQUAvl2+z+JqREREijcFlQJwe6towLz9k5yeaXE1IiIixZeCSgG4smYk1SIDSUzL1JwqIiIil0FBpQDY7TZubWm2quj2j4iISN4pqBSQm1tUwWG3sWLPKbYdSbC6HBERkWJJQaWAlA/15+q65QCYolYVERGRPFFQKUD9W5u3f35YuZ+kNHWqFRERuVQKKgWoc91yVI8MJC4lg+9XqFVFRETkUimoFCCH3ca9V9UA4LNFu3C5NVOtiIjIpVBQKWA3t4gmItCXfSdTmPPPYavLERERKVYUVApYgJ+DO6+sBsAnf+60uBoREZHiRUGlENzdtjp+Djur98aycs9Jq8sREREpNhRUCkHZECd9m1cG4OM/1KoiIiJysRRUCsl9HcxOtb9uPMLu40kWVyMiIlI8KKgUkpjyIVxdtyyGAZ8v2mV1OSIiIsWCgkohur9DTQC+W7GPU0npFlcjIiJS9CmoFKK2tSJpWCmU1Aw3X/+9x+pyREREijwFlUJks9k8rSoTF+8hNcNlcUUiIiJFm4JKIevZpCKVwvw5npjGt3pYoYiIyHkpqBQyX4edh66uDcD7C7aTkq5WFRERkXNRULHAbS2jiS4TwLGENCYt2W11OSIiIkWWgooF/HzsjLi2DgAfLtxBfGqGxRWJiIgUTQoqFunTvDK1ywUTm5zBZ39qXhUREZHcKKhYxGG38fh1ZqvKp3/u5KTmVREREclBQcVC3RpWoGGlUJLSXXy4cIfV5YiIiBQ5CioWstttPHl9XQAmLd7NkfhUiysSEREpWiwNKmPGjKFVq1aEhIRQrlw5+vTpw5YtW6wsqdB1rluWltUiSMt08978bVaXIyIiUqRYGlQWLlzI0KFDWbp0KXPnziUjI4Prr7+epKTS83Rhm83GU13NVpVvlu1l8+F4iysSEREpOmyGYRhWF5Hl2LFjlCtXjoULF9KxY8cLHh8fH09YWBhxcXGEhoYWQoUF5+GvVzJr/WGurFmGb+6/EpvNZnVJIiIiBeJS/n4XqT4qcXFxAJQpUybX/WlpacTHx3u9Sop/96iP08fO0p0nmbX+sNXliIiIFAlFJqi43W5GjBhB+/btadSoUa7HjBkzhrCwMM8rOjq6kKssOFUiAnmwUy0AXpu1SVPri4iIUISCytChQ9mwYQNTpkw55zHPPvsscXFxnte+fSXroX4PdqpFpTB/DsSm8PEfO60uR0RExHJFIqgMGzaMn3/+mQULFlClSpVzHud0OgkNDfV6lSQBfg7+3bM+AOMXbudAbIrFFYmIiFjL0qBiGAbDhg1j2rRp/Pbbb9SoUcPKcoqEno0r0qZGGVIz3Dw3bT1FqK+ziIhIobM0qAwdOpSvvvqKyZMnExISwuHDhzl8+DApKaW3JcFms/Fq38b4Oez8vuUY09ccsLokERERy1gaVMaPH09cXBydO3emYsWKnte3335rZVmWq10umEe7xAAw+qeNHE9Ms7giERERa1h+6ye316BBg6wsq0gY0rEmDSqGEpucwYsz/rG6HBEREUsUic60kpOvw87/3dwEh93GzHWHmPOP5lYREZHSR0GlCGtUOYwhHWsC8ML0DcSlZFhckYiISOFSUCniHr02hppRQRxNSOPVmRutLkdERKRQKagUcf6+Dt64uQk2G3y3Yj9/bTtudUkiIiKFRkGlGGhVvQx3X1kNgGemriMpLdPiikRERAqHgkox8VS3elQOD2D/qRRG//SPJoITEZFSQUGlmAh2+vB6v8aeW0DjF+6wuiQREZECp6BSjHSIKcvoGxsC8NacLSzbddLiikRERAqWgkoxc9eV1bipeWXcBjzyzWpOaNZaEREpwRRUihmbzcbLfRpRq2wQh+NTefy7tbjd6q8iIiIlk4JKMRTk9OGDAVfg9LGzcOsx/jN/m9UliYiIFAgFlWKqXoVQXu3bGID/zN/G7PWHLK5IREQk/ymoFGM3t6jCPe1rAPD4d2vZdCje4opERETyl4JKMffvHvXoEBNFSoaL+yat4GhCqtUliYiI5BsFlWLOx2FnXP/m1IgK4kBsCvdMXE6iZq4VEZESQkGlBAgP9GPi4FZEBvmx4UA8D321ktQMl9VliYiIXDYFlRKiWmQQnw9qRaCfgz+3HWfY5FWkZ7qtLktEROSyKKiUIE2jw/l0YEucPnbmbTrKY9+uIdOlsCIiIsWXgkoJ065WFB/d1QI/h52Z6w/x1A/rcGlCOBERKaYUVEqgznXL8f4dzfGx25i2+gDPTVuv2WtFRKRYUlApoa5vWIGxtzfDboMpy/cx+qd/MAyFFRERKV4UVEqwG5pU4q1bmmKzwaQlexgze7PVJYmIiFwSBZUS7qYrqvDa6an2P/5jJ18u3WNxRSIiIhdPQaUU6N+6Kv/qVheAF6ZvYOSPG8jQaCARESkGFFRKiYc61eK+q8znAn2xZA+DJiwjLjnD4qpERETOT0GllLDZbDx/QwM+G9iSQD8Hi7af4Kbxi9hzIsnq0kRERM5JQaWUubZ+eX54sB0Vw/zZcSyJvv9dzIrdJ60uS0REJFcKKqVQg0qhTB/ansaVwziZlM4dn/zNj2sOWF2WiIhIDgoqpVT5UH++feBKrm9QnnSXm0enrOGdX7eQlqmHGYqISNGhoFKKBfr58OGdLXigU00A3vttO9e+vZC9J5ItrkxERMSkoFLK2e02nu1en7dvaUpUsJP9p1K4afxivl2+1+rSREREFFTE1K9FFWY+chVVywRyPDGNp/+3nhd/3EBSWqbVpYmISCmmoCIe5UP9+WVEBx69NgYwp93v+d6frN0Xa21hIiJSaimoiJdAPx8eu64Onw9qSaUwf3afSKb3B4t4/Ls1HIpLsbo8EREpZRRUJFfX1CvPzEc60Ld5ZQCmrjrA1W/9zoRFu3C59RRmEREpHDbDMIrtX534+HjCwsKIi4sjNDTU6nJKrLX7Ynll5kaW7z4FQO1ywbzRrzEtqpWxuDIRESmOLuXvt4KKXBTDMPhq6R7enruV2NPPCOpYpyx3tqnKdQ3KY7PZLK5QRESKCwUVKTBxyRm8Omsj36/cT9ZvTvvakYzq1ZCY8iHWFiciIsWCgooUuF3Hk5iyfC8TFu0mPdONw26jVfUIBrWrQdeGamEREZFzU1CRQrP3RDIvz9zI3I1HPNva145kaOfatK0VqcAiIiI5KKhIodt+NIH/rTrAZ3/uIt3lBqBehRAGtatO72aVCfBzWFyhiIgUFQoqYpm9J5L55M+d/G/VfpLTzQcchvj70LtZJQa1q0HtcsEWVygiIlZTUBHLxaVk8P2KfUxaspt9J82J4uw2aFw5jFtaRtO7WSVC/H0trlJERKygoCJFhtttsGTnCSYs2sW8TUc9230dNq5vUIHbWkVzVe0o7Hb1ZRERKS0UVKRI2nkskd82H2Xy33vZeTzJs71yeACd65alZfUIujeqiL+v+rOIiJRkCipS5P1zMI7vlu9j2uoDxKdmP6HZz8dO6+pl6BATxVUxUdSvEKrWFhGREkZBRYqN1AwXv248woYDccxcd4gDsd4PPowM8qN97Siua1CeHo0r4lBoEREp9hRUpFgyDIMdxxL5Y+tx/tp+nKU7T3hGDgH4+9qpUz6ERpXDaFcrkpbVylAhzN/CikVEJC8UVKRESM90s3rvKX7feoyvl+7xukWUpWmVMBpUCqVCaADVowK5smYk5UMVXkREijIFFSlxUjNc7D+VzIYD8fxzMI75m46y+0QS7lx+eyuHB1CnfDBNqoTTukYZmkaHE+Dr0G0jEZEiQkFFSoWjCaks2HyUg7GpHI5LZeOheDYcjCO332gfu43mVcNpUiWcciFOWlaPoEJYAFHBfjh9NMpIRKQwKahIqRWXksHmQ/FsOZLAyj2nWL7rJAfjUs95vJ/DTu1ywdjt0CGmLA0rmb9HTauEUyHMH1+HvbBKFxEpNRRURM4Ql5LBqaR0luw8wa7jSew+nsTKPaeIT80gw3XuX3+bDcoGOykb4iQq2Em9CiFUCg+gZtkg/H0d+NhtVA4PoJz6xIiIXJJL+fvtU0g1iVgmLMCXsABfqkcFeW03DIN9J1PYdDiehNRMFmw+yr5TyaRnutl5LIl0l5ujCWkcTUgDYOHWYzk+22aDGlFBhAX4Ui7ESeXwQKpEBFA5IoBKYQGEB/oSEeRHkJ9DT5IWEckDBRUptWw2G1UjA6kaGQjAzS2qePa53QYnk9M5FJvK8cQ0DsensulQPAdjU9lzIokMl5v0TDcH41LZeSzpXKfw8HPYzdAS6EdEkC/VygThcNioViaQ8EBfQvx9CfH3OeOnD6H+vjh97Ao4IlKqKaiI5MJutxEVbN7yOZ8DsSnsOZ5EfGoGh+NSORCbYr5OpXA4PpVTyRmkZ7pztM4s3Xnyourwddi8wkuIMzvQVA73p1a5YKLLBOJyG0QG+WE/HWpCA3yJCPRVyBGRYk9BReQyVA4PoHJ4wDn3G4ZBSoaLU8lmP5lTyekcT0xj9/FkXG6D/aeSiU/NJCE1g4TUTBJSM4lPzSAxLRPDgAyXwcmkdE4mpV9ybWEBvlSJCCA2OQO7HQJ8HQT4+RDgayfA10H5UH8qhQfgsNuw2cDXbqdsiJMAPwdlQ5z42u2EBvjg52PH5TbIcBkEO32ICPTFR52MRaSQKKiIFCCbzUagnw+Bfj7nDTRnc7sNktIzPeElK8jEn/Fz74lkdhxL5MCpFHwcdk6dDjMGkJSeSVxKBnEpGQXwnSA8wJfQAF/chkGmy6BMkB9lgvzwsdsIzmoBcvoQ7PTBbrfhsNuoGOZPSroLh91GiL8PAX4+pKS7yHS7KRfij9PHTqbbTaCfD1XLBOI6/dkut0GQ00GAr/r5iJRGCioiRZDdnnXLxzdP70/NcLHnRDJ7TiQREeSHw24jNd1FcrqLlAwXKeku9pxM4mRSBm63gdswzNtT8WkkZ7g4npCGy20Qn5pBpsswW1wcdpLSzZaeU8kZnErODkGHzjMEPL/YbRDsNEOfn48dX4cNX4f99PIZ647T66eP8aw77Pj6nLXusJ3x/rOOz+X9fj62M449fa7T23zsthxBKtPlBlALlMhlUFARKYH8fR3UrRBC3Qoh+fq5LrfBqeR0TiSmk5CagcNuw8duZ+/JZFIyXLgNg4TUTBJTM0lMM1t/Mt0GKekuTiWnE+jnIMNlkJSWSVK6i0A/BzbgeGIaqRlu3IYZjlIz3DnO7TYgPjUz10cpFBV+WYHJx44NM9DZbVDpdGuay23eCgx2mv2MAnztHEtMw8duJ9DPQaCfeXvO5Xbj7+MgNMAXwzAwALvNbInKcBmkZ7oJDfChUlgA6S43brfhecq43WbeyrOdXnbYc3nZbPg4bNhtNnzsNuz27J8Omxm8IoP9Tt9+dJPpNnDYbNjteN5vs2V/1pnb7Xbzc+02cPo48Pc1O4S73IZmh5Y8KRJB5YMPPuDNN9/k8OHDNG3alHHjxtG6dWuryxKRszjO0cm4cZWwfDtH1ogqH4cZguw2SE53kZiWSWJaJslpLjLcbjIy3WS4DPN4l5uMrFem4b1++g+717rLfH/WcemZxhn73aS7jNOff9Z7XNnnTXflDFPpLjfpLuCMh2m6Ddh/yvup4LHJGYD3tpIqq/9TusuN7+l/pm7DbMWzYYYcu+3MkGP+ntltNsICfAn0c5Ca4SY107ymdpsNW9bnOuyE+Ju3GBNOB9gAPweGYYZCgCCnOfN0QmomNltWXy0Hfg47yekuUjPd+DnsOH3MFzbz1quB2YLn67B7OqknpmVgGBAR5IcNwAY2soNh1vc9exunv1+gn4NAp4OU07/PkcFOT2gMC/Al8fRt3bAAX895sz7HdvqENpv3Ncg6H57jzth3Zh1nrdts3ss+ZwTZM0Orw24jyOlDmSC/gvsluQDLg8q3337L448/zocffkibNm0YO3YsXbt2ZcuWLZQrV87q8kSkkGXdVjlTkNOHIKcP5S2qKTeGYZDpNs4RjsxWiKhgJ+mZbvafSvF0Wg5x+pCQZvY9Sk7LpGyIE7cByemZJJ++PeewQ2Kai5T0TM8fl6zWqqzbXUfiUzmVlI6fj/kHzTDAwMAwzHBkYOB2G7gM8w9vptuNy21+Tqbb8GxzuzH3GeA6fUxaposTien42M3WFYfdhtsw+wtl/TSXzUDgMrLOZeR4hIVh4Al1ZrB0nbkXXJxT1ig5sVavppUY17+5Zee3fGbaNm3a0KpVK95//30A3G430dHRDB8+nGeeeea879XMtCIiRYthmAHGbRikZbpJSXeR7nLj72M/3ZJleG4VGaePc7sxw84ZgcflNke8pWW4CfBznJ5TyAxh7tMhKd3lJinNvNUY6HTgsNlIyXBhs4HDbjdH3aW7MDDDLkDq6X5aaZkuAv188Pd1eFrx0jJdGEb27bP41Excbjduw2x5CPB1YAAJqRmng2HWd84OiWdehzP3uwyD5DQzjPr62An19+V4YhqGYQbF2OQMzzQE8SmZZog0gDM+2zjj+hqnP9g46/xnHnd2bcaZdZ31eZmnw3VuQbRnk4q8dUvTfP09KTYz06anp7Ny5UqeffZZzza73U6XLl1YsmRJjuPT0tJIS8tO2PHx8YVSp4iIXBybzYbDBg7M1phgp+UN91LMWdoV/fjx47hcLsqX927QLV++PIcPH85x/JgxYwgLC/O8oqOjC6tUERERsUCxGjP37LPPEhcX53nt27fP6pJERESkAFnaJhcVFYXD4eDIkSNe248cOUKFChVyHO90OnE6zz+luYiIiJQclrao+Pn50aJFC+bPn+/Z5na7mT9/Pm3btrWwMhERESkKLO/l9PjjjzNw4EBatmxJ69atGTt2LElJSQwePNjq0kRERMRilgeV2267jWPHjjFy5EgOHz5Ms2bN+OWXX3J0sBUREZHSx/J5VC6H5lEREREpfi7l73exGvUjIiIipYuCioiIiBRZCioiIiJSZCmoiIiISJGloCIiIiJFloKKiIiIFFkKKiIiIlJkWT7h2+XImgImPj7e4kpERETkYmX93b6YqdyKdVBJSEgAIDo62uJKRERE5FIlJCQQFhZ23mOK9cy0brebgwcPEhISgs1my9fPjo+PJzo6mn379mnW2wKk61w4dJ0Lj6514dB1LjwFca0NwyAhIYFKlSpht5+/F0qxblGx2+1UqVKlQM8RGhqqfwkKga5z4dB1Ljy61oVD17nw5Pe1vlBLShZ1phUREZEiS0FFREREiiwFlXNwOp28+OKLOJ1Oq0sp0XSdC4euc+HRtS4cus6Fx+prXaw704qIiEjJphYVERERKbIUVERERKTIUlARERGRIktBRURERIosBZVcfPDBB1SvXh1/f3/atGnDsmXLrC6pWPnjjz/o1asXlSpVwmazMX36dK/9hmEwcuRIKlasSEBAAF26dGHbtm1ex5w8eZIBAwYQGhpKeHg49957L4mJiYX4LYq+MWPG0KpVK0JCQihXrhx9+vRhy5YtXsekpqYydOhQIiMjCQ4Opl+/fhw5csTrmL1799KzZ08CAwMpV64cTz31FJmZmYX5VYq88ePH06RJE8+EV23btmX27Nme/brOBeP111/HZrMxYsQIzzZd68s3atQobDab16tevXqe/UXuGhviZcqUKYafn5/x+eefG//8849x//33G+Hh4caRI0esLq3YmDVrlvHcc88ZU6dONQBj2rRpXvtff/11IywszJg+fbqxdu1a48YbbzRq1KhhpKSkeI7p1q2b0bRpU2Pp0qXGn3/+adSuXdvo379/IX+Toq1r167GhAkTjA0bNhhr1qwxevToYVStWtVITEz0HPPggw8a0dHRxvz5840VK1YYV155pdGuXTvP/szMTKNRo0ZGly5djNWrVxuzZs0yoqKijGeffdaKr1RkzZgxw5g5c6axdetWY8uWLca///1vw9fX19iwYYNhGLrOBWHZsmVG9erVjSZNmhiPPvqoZ7uu9eV78cUXjYYNGxqHDh3yvI4dO+bZX9SusYLKWVq3bm0MHTrUs+5yuYxKlSoZY8aMsbCq4uvsoOJ2u40KFSoYb775pmdbbGys4XQ6jW+++cYwDMPYuHGjARjLly/3HDN79mzDZrMZBw4cKLTai5ujR48agLFw4ULDMMzr6uvra3z//feeYzZt2mQAxpIlSwzDMEOl3W43Dh8+7Dlm/PjxRmhoqJGWlla4X6CYiYiIMD799FNd5wKQkJBgxMTEGHPnzjU6derkCSq61vnjxRdfNJo2bZrrvqJ4jXXr5wzp6emsXLmSLl26eLbZ7Xa6dOnCkiVLLKys5Ni1axeHDx/2usZhYWG0adPGc42XLFlCeHg4LVu29BzTpUsX7HY7f//9d6HXXFzExcUBUKZMGQBWrlxJRkaG17WuV68eVatW9brWjRs3pnz58p5junbtSnx8PP/8808hVl98uFwupkyZQlJSEm3bttV1LgBDhw6lZ8+eXtcU9Dudn7Zt20alSpWoWbMmAwYMYO/evUDRvMbF+qGE+e348eO4XC6viw9Qvnx5Nm/ebFFVJcvhw4cBcr3GWfsOHz5MuXLlvPb7+PhQpkwZzzHize12M2LECNq3b0+jRo0A8zr6+fkRHh7udezZ1zq3fxZZ+yTb+vXradu2LampqQQHBzNt2jQaNGjAmjVrdJ3z0ZQpU1i1ahXLly/PsU+/0/mjTZs2TJw4kbp163Lo0CFGjx5Nhw4d2LBhQ5G8xgoqIiXA0KFD2bBhA3/99ZfVpZRYdevWZc2aNcTFxfHDDz8wcOBAFi5caHVZJcq+fft49NFHmTt3Lv7+/laXU2J1797ds9ykSRPatGlDtWrV+O677wgICLCwstzp1s8ZoqKicDgcOXo3HzlyhAoVKlhUVcmSdR3Pd40rVKjA0aNHvfZnZmZy8uRJ/XPIxbBhw/j5559ZsGABVapU8WyvUKEC6enpxMbGeh1/9rXO7Z9F1j7J5ufnR+3atWnRogVjxoyhadOm/Oc//9F1zkcrV67k6NGjXHHFFfj4+ODj48PChQt577338PHxoXz58rrWBSA8PJw6deqwffv2Ivn7rKByBj8/P1q0aMH8+fM929xuN/Pnz6dt27YWVlZy1KhRgwoVKnhd4/j4eP7++2/PNW7bti2xsbGsXLnSc8xvv/2G2+2mTZs2hV5zUWUYBsOGDWPatGn89ttv1KhRw2t/ixYt8PX19brWW7ZsYe/evV7Xev369V7BcO7cuYSGhtKgQYPC+SLFlNvtJi0tTdc5H1177bWsX7+eNWvWeF4tW7ZkwIABnmVd6/yXmJjIjh07qFixYtH8fc737rnF3JQpUwyn02lMnDjR2LhxozFkyBAjPDzcq3eznF9CQoKxevVqY/Xq1QZgvPPOO8bq1auNPXv2GIZhDk8ODw83fvzxR2PdunVG7969cx2e3Lx5c+Pvv/82/vrrLyMmJkbDk8/y0EMPGWFhYcbvv//uNcwwOTnZc8yDDz5oVK1a1fjtt9+MFStWGG3btjXatm3r2Z81zPD666831qxZY/zyyy9G2bJlNZTzLM8884yxcOFCY9euXca6deuMZ555xrDZbMavv/5qGIauc0E6c9SPYeha54cnnnjC+P33341du3YZixYtMrp06WJERUUZR48eNQyj6F1jBZVcjBs3zqhatarh5+dntG7d2li6dKnVJRUrCxYsMIAcr4EDBxqGYQ5RfuGFF4zy5csbTqfTuPbaa40tW7Z4fcaJEyeM/v37G8HBwUZoaKgxePBgIyEhwYJvU3Tldo0BY8KECZ5jUlJSjIcfftiIiIgwAgMDjb59+xqHDh3y+pzdu3cb3bt3NwICAoyoqCjjiSeeMDIyMgr52xRt99xzj1GtWjXDz8/PKFu2rHHttdd6Qoph6DoXpLODiq715bvtttuMihUrGn5+fkblypWN2267zdi+fbtnf1G7xjbDMIz8b6cRERERuXzqoyIiIiJFloKKiIiIFFkKKiIiIlJkKaiIiIhIkaWgIiIiIkWWgoqIiIgUWQoqIiIiUmQpqIhIiWKz2Zg+fbrVZYhIPlFQEZF8M2jQIGw2W45Xt27drC5NRIopH6sLEJGSpVu3bkyYMMFrm9PptKgaESnu1KIiIvnK6XRSoUIFr1dERARg3pYZP3483bt3JyAggJo1a/LDDz94vX/9+vVcc801BAQEEBkZyZAhQ0hMTPQ65vPPP6dhw4Y4nU4qVqzIsGHDvPYfP36cvn37EhgYSExMDDNmzCjYLy0iBUZBRUQK1QsvvEC/fv1Yu3YtAwYM4Pbbb2fTpk0AJCUl0bVrVyIiIli+fDnff/898+bN8woi48ePZ+jQoQwZMoT169czY8YMateu7XWO0aNHc+utt7Ju3Tp69OjBgAEDOHnyZKF+TxHJJwXyqEMRKZUGDhxoOBwOIygoyOv16quvGoZhPvH5wQcf9HpPmzZtjIceesgwDMP4+OOPjYiICCMxMdGzf+bMmYbdbjcOHz5sGIZhVKpUyXjuuefOWQNgPP/88571xMREAzBmz56db99TRAqP+qiISL66+uqrGT9+vNe2MmXKeJbbtm3rta9t27asWbMGgE2bNtG0aVOCgoI8+9u3b4/b7WbLli3YbDYOHjzItddee94amjRp4lkOCgoiNDSUo0eP5vUriYiFFFREJF8FBQXluBWTXwICAi7qOF9fX691m82G2+0uiJJEpICpj4qIFKqlS5fmWK9fvz4A9evXZ+3atSQlJXn2L1q0CLvdTt26dQkJCaF69erMnz+/UGsWEeuoRUVE8lVaWhqHDx/22ubj40NUVBQA33//PS1btuSqq67i66+/ZtmyZXz22WcADBgwgBdffJGBAwcyatQojh07xvDhw7nrrrsoX748AKNGjeLBBx+kXLlydO/enYSEBBYtWsTw4cML94uKSKFQUBGRfPXLL79QsWJFr21169Zl8+bNgDkiZ8qUKTz88MNUrFiRb775hgYNGgAQGBjInDlzePTRR2nVqhWBgYH069ePd955x/NZAwcOJDU1lXfffZcnn3ySqKgobr755sL7giJSqGyGYRhWFyEipYPNZmPatGn06dPH6lJEpJhQHxUREREpshRUREREpMhSHxURKTS60ywil0otKiIiIlJkKaiIiIhIkaWgIiIiIkWWgoqIiIgUWQoqIiIiUmQpqIiIiEiRpaAiIiIiRZaCioiIiBRZCioiIiJSZP0/pj7u4cRxLTsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# TODO : Plot History\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(t_loss, label='Training Loss')\n",
    "plt.plot(ev_loss, label='Validation Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Training and Validation Loss')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:07:20.743901Z",
     "iopub.status.busy": "2024-11-23T14:07:20.743285Z",
     "iopub.status.idle": "2024-11-23T14:07:20.749843Z",
     "shell.execute_reply": "2024-11-23T14:07:20.748837Z",
     "shell.execute_reply.started": "2024-11-23T14:07:20.743872Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English: In the beginning, God created the heavens and the earth.\n",
      "Urdu: ابتداء میں، خدا نے آسمانوں اور زمین کو پیدا کیا۔\n",
      "\n",
      "English: For God so loved the world that He gave His only begotten Son.\n",
      "Urdu: کیونکہ خدا نے دنیا سے اتنی محبت کی کہ اس نے اپنا اکلوتا بیٹا دیا۔\n",
      "\n",
      "English: The Lord is my shepherd; I shall not want.\n",
      "Urdu: خداوند میرا چرواہا ہے؛ مجھے کمی نہیں ہوگی۔\n",
      "\n",
      "English: Blessed are the meek, for they shall inherit the earth.\n",
      "Urdu: خوش قسمت ہیں وہ جو نرم دل ہیں، کیونکہ وہ زمین کے وارث ہوں گے۔\n",
      "\n",
      "English: I can do all things through Christ who strengthens me.\n",
      "Urdu: میں ہر کام کرسکتا ہوں مسیح کے ذریعے جو مجھے طاقت دیتا ہے۔\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Save the source (English) tokenizer\n",
    "# ar_tokenizer.save(\"eng_tokenizer.json\")\n",
    "\n",
    "# # Save the target (Urdu) tokenizer\n",
    "# en_tokenizer.save(\"urdu_tokenizer.json\")\n",
    "\n",
    "\n",
    "# TODO : translation\n",
    "def translate_sentence(\n",
    "    sentence: str,\n",
    "    src_tokenizer: Tokenizer,\n",
    "    trg_tokenizer: Tokenizer,\n",
    "    model: nn.Module,\n",
    "    device: torch.device,\n",
    "    max_len: int = 50):\n",
    "  \n",
    "    model.eval()\n",
    "    \n",
    "    # encode sentence\n",
    "    src = src_tokenizer.encode(sentence)\n",
    "    # get src input as ids and attention_mask\n",
    "    src_input = torch.tensor(src.ids).unsqueeze(0).to(device)\n",
    "    src_mask = model.make_src_mask(torch.tensor(src.attention_mask).unsqueeze(0).to(device))\n",
    "  \n",
    "    \n",
    "    \n",
    "    with torch.no_grad():\n",
    "        enc_src = model.encoder(src_input, src_mask)\n",
    "        \n",
    "    trg_indices = [trg_tokenizer.token_to_id(\"[SOS]\")]\n",
    "    \n",
    "    for i in range(max_len):\n",
    "        trg_inputs = torch.tensor(trg_indices).unsqueeze(0).to(device)\n",
    "\n",
    "        trg_mask = [1 if token != trg_tokenizer.token_to_id(\"[PAD]\") else 0 for token in trg_indices]\n",
    "        trg_mask = torch.tensor(trg_mask).unsqueeze(0).to(device)\n",
    "        trg_mask = model.make_trg_mask(trg_mask)\n",
    "        \n",
    "        output, attention = model.decoder(trg_inputs, enc_src, trg_mask, src_mask)\n",
    "        \n",
    "        pred_token = output.argmax(2)[:,-1].item()\n",
    "        \n",
    "        trg_indices.append(pred_token)\n",
    "\n",
    "        if pred_token == trg_tokenizer.token_to_id(\"[EOS]\"):\n",
    "            break\n",
    "        \n",
    "            \n",
    "    return trg_tokenizer.decode(trg_indices), attention\n",
    "    \n",
    "\n",
    "# Example list of English sentences to translate\n",
    "english_sentences = [\n",
    "    \"In the beginning, God created the heavens and the earth.\",\n",
    "    \"For God so loved the world that He gave His only begotten Son.\",\n",
    "    \"The Lord is my shepherd; I shall not want.\",\n",
    "    \"Blessed are the meek, for they shall inherit the earth.\",\n",
    "    \"I can do all things through Christ who strengthens me.\"\n",
    "]\n",
    "\n",
    "\n",
    "# en_tokenizer = Tokenizer.from_file(\"/kaggle/input/weights/pytorch/default/1/eng_tokenizer.json\")\n",
    "# ur_tokenizer = Tokenizer.from_file(\"/kaggle/input/weights/pytorch/default/1/urdu_tokenizer.json\")\n",
    "\n",
    "# Translate each sentence and collect results\n",
    "urdu_translations = []\n",
    "for sentence in english_sentences:\n",
    "    translation, atn = translate_sentence(\n",
    "        sentence=sentence,\n",
    "        src_tokenizer=en_tokenizer,\n",
    "        trg_tokenizer=ur_tokenizer,\n",
    "        model=model,\n",
    "        device=var.device,\n",
    "        max_len=20\n",
    "    )\n",
    "    urdu_translations.append(translation)\n",
    "\n",
    "# Display translations\n",
    "for en, ur in zip(english_sentences, urdu_translations):\n",
    "    print(f\"English: {en}\")\n",
    "    print(f\"Urdu: {ur}\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:27:35.795770Z",
     "iopub.status.busy": "2024-11-23T14:27:35.795437Z",
     "iopub.status.idle": "2024-11-23T14:27:35.812182Z",
     "shell.execute_reply": "2024-11-23T14:27:35.811335Z",
     "shell.execute_reply.started": "2024-11-23T14:27:35.795747Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 1, Train Loss: 5.8641, Val Loss: 5.3884\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 2, Train Loss: 5.8525, Val Loss: 5.3777\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 3, Train Loss: 5.8408, Val Loss: 5.3670\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 4, Train Loss: 5.8292, Val Loss: 5.3564\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 5, Train Loss: 5.8176, Val Loss: 5.3457\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 6, Train Loss: 5.8060, Val Loss: 5.3350\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 7, Train Loss: 5.7943, Val Loss: 5.3243\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 8, Train Loss: 5.7827, Val Loss: 5.3137\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 9, Train Loss: 5.7711, Val Loss: 5.3030\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 10, Train Loss: 5.7594, Val Loss: 5.2923\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 11, Train Loss: 5.7478, Val Loss: 5.2816\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 12, Train Loss: 5.7362, Val Loss: 5.2710\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 13, Train Loss: 5.7246, Val Loss: 5.2603\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 14, Train Loss: 5.7129, Val Loss: 5.2496\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 15, Train Loss: 5.7013, Val Loss: 5.2389\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 16, Train Loss: 5.6897, Val Loss: 5.2282\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 17, Train Loss: 5.6780, Val Loss: 5.2176\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 18, Train Loss: 5.6664, Val Loss: 5.2069\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 19, Train Loss: 5.6548, Val Loss: 5.1962\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 20, Train Loss: 5.6432, Val Loss: 5.1855\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 21, Train Loss: 5.6315, Val Loss: 5.1749\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 22, Train Loss: 5.6199, Val Loss: 5.1642\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 23, Train Loss: 5.6083, Val Loss: 5.1535\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 24, Train Loss: 5.5967, Val Loss: 5.1428\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 25, Train Loss: 5.5850, Val Loss: 5.1322\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 26, Train Loss: 5.5734, Val Loss: 5.1215\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 27, Train Loss: 5.5618, Val Loss: 5.1108\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 28, Train Loss: 5.5501, Val Loss: 5.1001\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 29, Train Loss: 5.5385, Val Loss: 5.0894\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 30, Train Loss: 5.5269, Val Loss: 5.0788\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 31, Train Loss: 5.5153, Val Loss: 5.0681\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 32, Train Loss: 5.5036, Val Loss: 5.0574\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 33, Train Loss: 5.4920, Val Loss: 5.0467\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 34, Train Loss: 5.4804, Val Loss: 5.0361\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 35, Train Loss: 5.4687, Val Loss: 5.0254\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 36, Train Loss: 5.4571, Val Loss: 5.0147\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 37, Train Loss: 5.4455, Val Loss: 5.0040\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 38, Train Loss: 5.4339, Val Loss: 4.9934\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 39, Train Loss: 5.4222, Val Loss: 4.9827\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 40, Train Loss: 5.4106, Val Loss: 4.9720\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 41, Train Loss: 5.3990, Val Loss: 4.9613\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 42, Train Loss: 5.3873, Val Loss: 4.9507\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 43, Train Loss: 5.3757, Val Loss: 4.9400\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 44, Train Loss: 5.3641, Val Loss: 4.9293\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 45, Train Loss: 5.3525, Val Loss: 4.9186\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 46, Train Loss: 5.3408, Val Loss: 4.9079\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 47, Train Loss: 5.3292, Val Loss: 4.8973\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 48, Train Loss: 5.3176, Val Loss: 4.8866\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 49, Train Loss: 5.3059, Val Loss: 4.8759\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 50, Train Loss: 5.2943, Val Loss: 4.8652\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 51, Train Loss: 5.2827, Val Loss: 4.8546\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 52, Train Loss: 5.2711, Val Loss: 4.8439\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 53, Train Loss: 5.2594, Val Loss: 4.8332\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 54, Train Loss: 5.2478, Val Loss: 4.8225\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 55, Train Loss: 5.2362, Val Loss: 4.8119\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 56, Train Loss: 5.2245, Val Loss: 4.8012\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 57, Train Loss: 5.2129, Val Loss: 4.7905\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 58, Train Loss: 5.2013, Val Loss: 4.7798\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 59, Train Loss: 5.1897, Val Loss: 4.7691\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 60, Train Loss: 5.1780, Val Loss: 4.7585\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 61, Train Loss: 5.1664, Val Loss: 4.7478\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 62, Train Loss: 5.1548, Val Loss: 4.7371\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 63, Train Loss: 5.1432, Val Loss: 4.7264\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 64, Train Loss: 5.1315, Val Loss: 4.7158\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 65, Train Loss: 5.1199, Val Loss: 4.7051\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 66, Train Loss: 5.1083, Val Loss: 4.6944\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 67, Train Loss: 5.0966, Val Loss: 4.6837\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 68, Train Loss: 5.0850, Val Loss: 4.6731\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 69, Train Loss: 5.0734, Val Loss: 4.6624\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 70, Train Loss: 5.0618, Val Loss: 4.6517\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 71, Train Loss: 5.0501, Val Loss: 4.6410\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 72, Train Loss: 5.0385, Val Loss: 4.6303\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 73, Train Loss: 5.0269, Val Loss: 4.6197\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 74, Train Loss: 5.0152, Val Loss: 4.6090\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 75, Train Loss: 5.0036, Val Loss: 4.5983\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 76, Train Loss: 4.9920, Val Loss: 4.5876\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 77, Train Loss: 4.9804, Val Loss: 4.5770\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 78, Train Loss: 4.9687, Val Loss: 4.5663\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 79, Train Loss: 4.9571, Val Loss: 4.5556\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 80, Train Loss: 4.9455, Val Loss: 4.5449\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 81, Train Loss: 4.9338, Val Loss: 4.5343\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 82, Train Loss: 4.9222, Val Loss: 4.5236\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 83, Train Loss: 4.9106, Val Loss: 4.5129\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 84, Train Loss: 4.8990, Val Loss: 4.5022\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 85, Train Loss: 4.8873, Val Loss: 4.4915\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 86, Train Loss: 4.8757, Val Loss: 4.4809\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 87, Train Loss: 4.8641, Val Loss: 4.4702\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 88, Train Loss: 4.8524, Val Loss: 4.4595\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 89, Train Loss: 4.8408, Val Loss: 4.4488\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 90, Train Loss: 4.8292, Val Loss: 4.4382\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 91, Train Loss: 4.8176, Val Loss: 4.4275\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 92, Train Loss: 4.8059, Val Loss: 4.4168\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 93, Train Loss: 4.7943, Val Loss: 4.4061\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 94, Train Loss: 4.7827, Val Loss: 4.3955\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 95, Train Loss: 4.7710, Val Loss: 4.3848\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 96, Train Loss: 4.7594, Val Loss: 4.3741\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 97, Train Loss: 4.7478, Val Loss: 4.3634\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 98, Train Loss: 4.7362, Val Loss: 4.3528\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 99, Train Loss: 4.7245, Val Loss: 4.3421\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 100, Train Loss: 4.7129, Val Loss: 4.3314\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 101, Train Loss: 4.7013, Val Loss: 4.3207\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 102, Train Loss: 4.6897, Val Loss: 4.3100\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 103, Train Loss: 4.6780, Val Loss: 4.2994\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 104, Train Loss: 4.6664, Val Loss: 4.2887\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 105, Train Loss: 4.6548, Val Loss: 4.2780\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 106, Train Loss: 4.6431, Val Loss: 4.2673\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 107, Train Loss: 4.6315, Val Loss: 4.2567\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 108, Train Loss: 4.6199, Val Loss: 4.2460\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 109, Train Loss: 4.6083, Val Loss: 4.2353\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 110, Train Loss: 4.5966, Val Loss: 4.2246\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 111, Train Loss: 4.5850, Val Loss: 4.2140\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 112, Train Loss: 4.5734, Val Loss: 4.2033\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 113, Train Loss: 4.5617, Val Loss: 4.1926\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 114, Train Loss: 4.5501, Val Loss: 4.1819\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 115, Train Loss: 4.5385, Val Loss: 4.1712\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 116, Train Loss: 4.5269, Val Loss: 4.1606\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 117, Train Loss: 4.5152, Val Loss: 4.1499\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 118, Train Loss: 4.5036, Val Loss: 4.1392\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 119, Train Loss: 4.4920, Val Loss: 4.1285\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 120, Train Loss: 4.4803, Val Loss: 4.1179\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 121, Train Loss: 4.4687, Val Loss: 4.1072\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 122, Train Loss: 4.4571, Val Loss: 4.0965\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 123, Train Loss: 4.4455, Val Loss: 4.0858\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 124, Train Loss: 4.4338, Val Loss: 4.0752\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 125, Train Loss: 4.4222, Val Loss: 4.0645\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 126, Train Loss: 4.4106, Val Loss: 4.0538\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 127, Train Loss: 4.3989, Val Loss: 4.0431\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 128, Train Loss: 4.3873, Val Loss: 4.0324\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 129, Train Loss: 4.3757, Val Loss: 4.0218\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 130, Train Loss: 4.3641, Val Loss: 4.0111\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 131, Train Loss: 4.3524, Val Loss: 4.0004\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 132, Train Loss: 4.3408, Val Loss: 3.9897\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 133, Train Loss: 4.3292, Val Loss: 3.9791\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 134, Train Loss: 4.3175, Val Loss: 3.9684\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 135, Train Loss: 4.3059, Val Loss: 3.9577\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 136, Train Loss: 4.2943, Val Loss: 3.9470\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 137, Train Loss: 4.2827, Val Loss: 3.9364\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 138, Train Loss: 4.2710, Val Loss: 3.9257\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 139, Train Loss: 4.2594, Val Loss: 3.9150\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 140, Train Loss: 4.2478, Val Loss: 3.9043\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 141, Train Loss: 4.2362, Val Loss: 3.8936\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 142, Train Loss: 4.2245, Val Loss: 3.8830\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 143, Train Loss: 4.2129, Val Loss: 3.8723\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 144, Train Loss: 4.2013, Val Loss: 3.8616\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 145, Train Loss: 4.1896, Val Loss: 3.8509\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 146, Train Loss: 4.1780, Val Loss: 3.8403\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 147, Train Loss: 4.1664, Val Loss: 3.8296\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 148, Train Loss: 4.1548, Val Loss: 3.8189\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 149, Train Loss: 4.1431, Val Loss: 3.8082\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 150, Train Loss: 4.1315, Val Loss: 3.7976\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 151, Train Loss: 4.1199, Val Loss: 3.7869\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 152, Train Loss: 4.1082, Val Loss: 3.7762\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 153, Train Loss: 4.0966, Val Loss: 3.7655\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 154, Train Loss: 4.0850, Val Loss: 3.7548\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 155, Train Loss: 4.0734, Val Loss: 3.7442\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 156, Train Loss: 4.0617, Val Loss: 3.7335\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 157, Train Loss: 4.0501, Val Loss: 3.7228\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 158, Train Loss: 4.0385, Val Loss: 3.7121\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 159, Train Loss: 4.0268, Val Loss: 3.7015\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 160, Train Loss: 4.0152, Val Loss: 3.6908\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 161, Train Loss: 4.0036, Val Loss: 3.6801\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 162, Train Loss: 3.9920, Val Loss: 3.6694\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 163, Train Loss: 3.9803, Val Loss: 3.6588\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 164, Train Loss: 3.9687, Val Loss: 3.6481\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 165, Train Loss: 3.9571, Val Loss: 3.6374\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 166, Train Loss: 3.9454, Val Loss: 3.6267\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 167, Train Loss: 3.9338, Val Loss: 3.6161\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 168, Train Loss: 3.9222, Val Loss: 3.6054\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 169, Train Loss: 3.9106, Val Loss: 3.5947\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 170, Train Loss: 3.8989, Val Loss: 3.5840\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 171, Train Loss: 3.8873, Val Loss: 3.5733\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 172, Train Loss: 3.8757, Val Loss: 3.5627\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 173, Train Loss: 3.8640, Val Loss: 3.5520\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 174, Train Loss: 3.8524, Val Loss: 3.5413\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 175, Train Loss: 3.8408, Val Loss: 3.5306\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 176, Train Loss: 3.8292, Val Loss: 3.5200\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 177, Train Loss: 3.8175, Val Loss: 3.5093\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 178, Train Loss: 3.8059, Val Loss: 3.4986\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 179, Train Loss: 3.7943, Val Loss: 3.4879\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 180, Train Loss: 3.7827, Val Loss: 3.4773\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 181, Train Loss: 3.7710, Val Loss: 3.4666\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 182, Train Loss: 3.7594, Val Loss: 3.4559\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 183, Train Loss: 3.7478, Val Loss: 3.4452\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 184, Train Loss: 3.7361, Val Loss: 3.4345\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 185, Train Loss: 3.7245, Val Loss: 3.4239\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 186, Train Loss: 3.7129, Val Loss: 3.4132\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 187, Train Loss: 3.7013, Val Loss: 3.4025\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 188, Train Loss: 3.6896, Val Loss: 3.3918\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 189, Train Loss: 3.6780, Val Loss: 3.3812\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 190, Train Loss: 3.6664, Val Loss: 3.3705\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 191, Train Loss: 3.6547, Val Loss: 3.3598\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 192, Train Loss: 3.6431, Val Loss: 3.3491\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 193, Train Loss: 3.6315, Val Loss: 3.3385\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 194, Train Loss: 3.6199, Val Loss: 3.3278\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 195, Train Loss: 3.6082, Val Loss: 3.3171\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 196, Train Loss: 3.5966, Val Loss: 3.3064\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 197, Train Loss: 3.5850, Val Loss: 3.2957\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 198, Train Loss: 3.5733, Val Loss: 3.2851\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 199, Train Loss: 3.5617, Val Loss: 3.2744\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 200, Train Loss: 3.5501, Val Loss: 3.2637\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 201, Train Loss: 3.5385, Val Loss: 3.2530\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 202, Train Loss: 3.5268, Val Loss: 3.2424\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 203, Train Loss: 3.5152, Val Loss: 3.2317\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 204, Train Loss: 3.5036, Val Loss: 3.2210\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 205, Train Loss: 3.4919, Val Loss: 3.2103\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 206, Train Loss: 3.4803, Val Loss: 3.1997\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 207, Train Loss: 3.4687, Val Loss: 3.1890\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 208, Train Loss: 3.4571, Val Loss: 3.1783\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 209, Train Loss: 3.4454, Val Loss: 3.1676\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 210, Train Loss: 3.4338, Val Loss: 3.1569\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 211, Train Loss: 3.4222, Val Loss: 3.1463\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 212, Train Loss: 3.4105, Val Loss: 3.1356\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 213, Train Loss: 3.3989, Val Loss: 3.1249\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 214, Train Loss: 3.3873, Val Loss: 3.1142\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 215, Train Loss: 3.3757, Val Loss: 3.1036\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 216, Train Loss: 3.3640, Val Loss: 3.0929\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 217, Train Loss: 3.3524, Val Loss: 3.0822\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 218, Train Loss: 3.3408, Val Loss: 3.0715\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 219, Train Loss: 3.3292, Val Loss: 3.0609\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 220, Train Loss: 3.3175, Val Loss: 3.0502\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 221, Train Loss: 3.3059, Val Loss: 3.0395\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 222, Train Loss: 3.2943, Val Loss: 3.0288\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 223, Train Loss: 3.2826, Val Loss: 3.0182\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 224, Train Loss: 3.2710, Val Loss: 3.0075\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 225, Train Loss: 3.2594, Val Loss: 2.9968\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 226, Train Loss: 3.2478, Val Loss: 2.9861\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 227, Train Loss: 3.2361, Val Loss: 2.9754\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 228, Train Loss: 3.2245, Val Loss: 2.9648\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 229, Train Loss: 3.2129, Val Loss: 2.9541\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 230, Train Loss: 3.2012, Val Loss: 2.9434\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 231, Train Loss: 3.1896, Val Loss: 2.9327\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 232, Train Loss: 3.1780, Val Loss: 2.9221\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 233, Train Loss: 3.1664, Val Loss: 2.9114\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 234, Train Loss: 3.1547, Val Loss: 2.9007\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 235, Train Loss: 3.1431, Val Loss: 2.8900\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 236, Train Loss: 3.1315, Val Loss: 2.8794\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 237, Train Loss: 3.1198, Val Loss: 2.8687\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 238, Train Loss: 3.1082, Val Loss: 2.8580\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 239, Train Loss: 3.0966, Val Loss: 2.8473\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 240, Train Loss: 3.0850, Val Loss: 2.8366\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 241, Train Loss: 3.0733, Val Loss: 2.8260\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 242, Train Loss: 3.0617, Val Loss: 2.8153\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 243, Train Loss: 3.0501, Val Loss: 2.8046\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 244, Train Loss: 3.0384, Val Loss: 2.7939\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 245, Train Loss: 3.0268, Val Loss: 2.7833\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 246, Train Loss: 3.0152, Val Loss: 2.7726\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 247, Train Loss: 3.0036, Val Loss: 2.7619\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 248, Train Loss: 2.9919, Val Loss: 2.7512\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 249, Train Loss: 2.9803, Val Loss: 2.7406\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 250, Train Loss: 2.9687, Val Loss: 2.7299\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 251, Train Loss: 2.9571, Val Loss: 2.7192\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 252, Train Loss: 2.9454, Val Loss: 2.7085\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 253, Train Loss: 2.9338, Val Loss: 2.6978\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 254, Train Loss: 2.9222, Val Loss: 2.6872\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 255, Train Loss: 2.9105, Val Loss: 2.6765\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 256, Train Loss: 2.8989, Val Loss: 2.6658\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 257, Train Loss: 2.8873, Val Loss: 2.6551\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 258, Train Loss: 2.8757, Val Loss: 2.6445\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 259, Train Loss: 2.8640, Val Loss: 2.6338\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 260, Train Loss: 2.8524, Val Loss: 2.6231\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 261, Train Loss: 2.8408, Val Loss: 2.6124\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 262, Train Loss: 2.8291, Val Loss: 2.6018\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 263, Train Loss: 2.8175, Val Loss: 2.5911\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 264, Train Loss: 2.8059, Val Loss: 2.5804\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 265, Train Loss: 2.7943, Val Loss: 2.5697\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 266, Train Loss: 2.7826, Val Loss: 2.5590\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 267, Train Loss: 2.7710, Val Loss: 2.5484\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 268, Train Loss: 2.7594, Val Loss: 2.5377\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 269, Train Loss: 2.7477, Val Loss: 2.5270\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 270, Train Loss: 2.7361, Val Loss: 2.5163\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 271, Train Loss: 2.7245, Val Loss: 2.5057\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 272, Train Loss: 2.7129, Val Loss: 2.4950\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 273, Train Loss: 2.7012, Val Loss: 2.4843\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 274, Train Loss: 2.6896, Val Loss: 2.4736\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 275, Train Loss: 2.6780, Val Loss: 2.4630\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 276, Train Loss: 2.6663, Val Loss: 2.4523\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 277, Train Loss: 2.6547, Val Loss: 2.4416\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 278, Train Loss: 2.6431, Val Loss: 2.4309\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 279, Train Loss: 2.6315, Val Loss: 2.4202\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 280, Train Loss: 2.6198, Val Loss: 2.4096\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 281, Train Loss: 2.6082, Val Loss: 2.3989\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 282, Train Loss: 2.5966, Val Loss: 2.3882\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 283, Train Loss: 2.5849, Val Loss: 2.3775\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 284, Train Loss: 2.5733, Val Loss: 2.3669\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 285, Train Loss: 2.5617, Val Loss: 2.3562\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 286, Train Loss: 2.5501, Val Loss: 2.3455\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 287, Train Loss: 2.5384, Val Loss: 2.3348\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 288, Train Loss: 2.5268, Val Loss: 2.3242\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 289, Train Loss: 2.5152, Val Loss: 2.3135\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 290, Train Loss: 2.5036, Val Loss: 2.3028\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 291, Train Loss: 2.4919, Val Loss: 2.2921\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 292, Train Loss: 2.4803, Val Loss: 2.2815\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 293, Train Loss: 2.4687, Val Loss: 2.2708\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 294, Train Loss: 2.4570, Val Loss: 2.2601\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 295, Train Loss: 2.4454, Val Loss: 2.2494\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 296, Train Loss: 2.4338, Val Loss: 2.2387\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 297, Train Loss: 2.4222, Val Loss: 2.2281\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 298, Train Loss: 2.4105, Val Loss: 2.2174\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 299, Train Loss: 2.3989, Val Loss: 2.2067\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 300, Train Loss: 2.3873, Val Loss: 2.1960\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 301, Train Loss: 2.3756, Val Loss: 2.1854\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 302, Train Loss: 2.3640, Val Loss: 2.1747\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 303, Train Loss: 2.3524, Val Loss: 2.1640\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 304, Train Loss: 2.3408, Val Loss: 2.1533\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 305, Train Loss: 2.3291, Val Loss: 2.1427\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 306, Train Loss: 2.3175, Val Loss: 2.1320\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 307, Train Loss: 2.3059, Val Loss: 2.1213\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 308, Train Loss: 2.2942, Val Loss: 2.1106\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 309, Train Loss: 2.2826, Val Loss: 2.0999\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 310, Train Loss: 2.2710, Val Loss: 2.0893\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 311, Train Loss: 2.2594, Val Loss: 2.0786\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 312, Train Loss: 2.2477, Val Loss: 2.0679\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 313, Train Loss: 2.2361, Val Loss: 2.0572\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 314, Train Loss: 2.2245, Val Loss: 2.0466\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 315, Train Loss: 2.2128, Val Loss: 2.0359\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 316, Train Loss: 2.2012, Val Loss: 2.0252\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 317, Train Loss: 2.1896, Val Loss: 2.0145\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 318, Train Loss: 2.1780, Val Loss: 2.0039\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 319, Train Loss: 2.1663, Val Loss: 1.9932\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 320, Train Loss: 2.1547, Val Loss: 1.9825\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 321, Train Loss: 2.1431, Val Loss: 1.9718\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 322, Train Loss: 2.1314, Val Loss: 1.9611\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 323, Train Loss: 2.1198, Val Loss: 1.9505\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 324, Train Loss: 2.1082, Val Loss: 1.9398\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 325, Train Loss: 2.0966, Val Loss: 1.9291\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 326, Train Loss: 2.0849, Val Loss: 1.9184\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 327, Train Loss: 2.0733, Val Loss: 1.9078\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 328, Train Loss: 2.0617, Val Loss: 1.8971\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 329, Train Loss: 2.0501, Val Loss: 1.8864\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 330, Train Loss: 2.0384, Val Loss: 1.8757\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 331, Train Loss: 2.0268, Val Loss: 1.8651\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 332, Train Loss: 2.0152, Val Loss: 1.8544\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 333, Train Loss: 2.0035, Val Loss: 1.8437\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 334, Train Loss: 1.9919, Val Loss: 1.8330\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 335, Train Loss: 1.9803, Val Loss: 1.8223\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 336, Train Loss: 1.9687, Val Loss: 1.8117\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 337, Train Loss: 1.9570, Val Loss: 1.8010\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 338, Train Loss: 1.9454, Val Loss: 1.7903\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 339, Train Loss: 1.9338, Val Loss: 1.7796\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 340, Train Loss: 1.9221, Val Loss: 1.7690\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 341, Train Loss: 1.9105, Val Loss: 1.7583\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 342, Train Loss: 1.8989, Val Loss: 1.7476\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 343, Train Loss: 1.8873, Val Loss: 1.7369\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 344, Train Loss: 1.8756, Val Loss: 1.7263\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 345, Train Loss: 1.8640, Val Loss: 1.7156\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 346, Train Loss: 1.8524, Val Loss: 1.7049\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 347, Train Loss: 1.8407, Val Loss: 1.6942\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 348, Train Loss: 1.8291, Val Loss: 1.6836\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 349, Train Loss: 1.8175, Val Loss: 1.6729\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 350, Train Loss: 1.8059, Val Loss: 1.6622\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 351, Train Loss: 1.7942, Val Loss: 1.6515\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 352, Train Loss: 1.7826, Val Loss: 1.6408\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 353, Train Loss: 1.7710, Val Loss: 1.6302\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 354, Train Loss: 1.7593, Val Loss: 1.6195\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 355, Train Loss: 1.7477, Val Loss: 1.6088\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 356, Train Loss: 1.7361, Val Loss: 1.5981\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 357, Train Loss: 1.7245, Val Loss: 1.5875\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 358, Train Loss: 1.7128, Val Loss: 1.5768\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 359, Train Loss: 1.7012, Val Loss: 1.5661\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 360, Train Loss: 1.6896, Val Loss: 1.5554\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 361, Train Loss: 1.6779, Val Loss: 1.5448\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 362, Train Loss: 1.6663, Val Loss: 1.5341\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 363, Train Loss: 1.6547, Val Loss: 1.5234\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 364, Train Loss: 1.6431, Val Loss: 1.5127\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 365, Train Loss: 1.6314, Val Loss: 1.5020\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 366, Train Loss: 1.6198, Val Loss: 1.4914\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 367, Train Loss: 1.6082, Val Loss: 1.4807\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 368, Train Loss: 1.5966, Val Loss: 1.4700\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 369, Train Loss: 1.5849, Val Loss: 1.4593\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 370, Train Loss: 1.5733, Val Loss: 1.4487\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 371, Train Loss: 1.5617, Val Loss: 1.4380\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 372, Train Loss: 1.5500, Val Loss: 1.4273\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 373, Train Loss: 1.5384, Val Loss: 1.4166\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 374, Train Loss: 1.5268, Val Loss: 1.4060\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 375, Train Loss: 1.5152, Val Loss: 1.3953\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 376, Train Loss: 1.5035, Val Loss: 1.3846\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 377, Train Loss: 1.4919, Val Loss: 1.3739\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 378, Train Loss: 1.4803, Val Loss: 1.3632\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 379, Train Loss: 1.4686, Val Loss: 1.3526\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 380, Train Loss: 1.4570, Val Loss: 1.3419\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 381, Train Loss: 1.4454, Val Loss: 1.3312\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 382, Train Loss: 1.4338, Val Loss: 1.3205\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 383, Train Loss: 1.4221, Val Loss: 1.3099\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 384, Train Loss: 1.4105, Val Loss: 1.2992\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 385, Train Loss: 1.3989, Val Loss: 1.2885\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 386, Train Loss: 1.3872, Val Loss: 1.2778\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 387, Train Loss: 1.3756, Val Loss: 1.2672\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 388, Train Loss: 1.3640, Val Loss: 1.2565\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 389, Train Loss: 1.3524, Val Loss: 1.2458\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 390, Train Loss: 1.3407, Val Loss: 1.2351\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 391, Train Loss: 1.3291, Val Loss: 1.2244\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 392, Train Loss: 1.3175, Val Loss: 1.2138\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 393, Train Loss: 1.3058, Val Loss: 1.2031\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 394, Train Loss: 1.2942, Val Loss: 1.1924\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 395, Train Loss: 1.2826, Val Loss: 1.1817\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 396, Train Loss: 1.2710, Val Loss: 1.1711\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 397, Train Loss: 1.2593, Val Loss: 1.1604\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 398, Train Loss: 1.2477, Val Loss: 1.1497\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 399, Train Loss: 1.2361, Val Loss: 1.1390\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 400, Train Loss: 1.2244, Val Loss: 1.1284\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 401, Train Loss: 1.2128, Val Loss: 1.1177\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 402, Train Loss: 1.2012, Val Loss: 1.1070\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 403, Train Loss: 1.1896, Val Loss: 1.0963\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 404, Train Loss: 1.1779, Val Loss: 1.0856\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 405, Train Loss: 1.1663, Val Loss: 1.0750\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 406, Train Loss: 1.1547, Val Loss: 1.0643\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 407, Train Loss: 1.1431, Val Loss: 1.0536\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 408, Train Loss: 1.1314, Val Loss: 1.0429\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 409, Train Loss: 1.1198, Val Loss: 1.0323\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 410, Train Loss: 1.1082, Val Loss: 1.0216\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 411, Train Loss: 1.0965, Val Loss: 1.0109\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 412, Train Loss: 1.0849, Val Loss: 1.0002\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 413, Train Loss: 1.0733, Val Loss: 0.9896\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 414, Train Loss: 1.0617, Val Loss: 0.9789\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 415, Train Loss: 1.0500, Val Loss: 0.9682\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 416, Train Loss: 1.0384, Val Loss: 0.9575\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 417, Train Loss: 1.0268, Val Loss: 0.9469\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 418, Train Loss: 1.0151, Val Loss: 0.9362\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 419, Train Loss: 1.0035, Val Loss: 0.9255\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 420, Train Loss: 0.9919, Val Loss: 0.9148\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 421, Train Loss: 0.9803, Val Loss: 0.9041\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 422, Train Loss: 0.9686, Val Loss: 0.8935\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 423, Train Loss: 0.9570, Val Loss: 0.8828\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 424, Train Loss: 0.9454, Val Loss: 0.8721\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 425, Train Loss: 0.9337, Val Loss: 0.8614\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 426, Train Loss: 0.9221, Val Loss: 0.8508\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 427, Train Loss: 0.9105, Val Loss: 0.8401\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 428, Train Loss: 0.8989, Val Loss: 0.8294\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 429, Train Loss: 0.8872, Val Loss: 0.8187\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 430, Train Loss: 0.8756, Val Loss: 0.8081\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 431, Train Loss: 0.8640, Val Loss: 0.7974\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 432, Train Loss: 0.8523, Val Loss: 0.7867\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 433, Train Loss: 0.8407, Val Loss: 0.7760\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 434, Train Loss: 0.8291, Val Loss: 0.7653\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 435, Train Loss: 0.8175, Val Loss: 0.7547\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 436, Train Loss: 0.8058, Val Loss: 0.7440\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 437, Train Loss: 0.7942, Val Loss: 0.7333\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 438, Train Loss: 0.7826, Val Loss: 0.7226\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 439, Train Loss: 0.7709, Val Loss: 0.7120\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 440, Train Loss: 0.7593, Val Loss: 0.7013\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 441, Train Loss: 0.7477, Val Loss: 0.6906\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 442, Train Loss: 0.7361, Val Loss: 0.6799\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 443, Train Loss: 0.7244, Val Loss: 0.6693\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 444, Train Loss: 0.7128, Val Loss: 0.6586\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 445, Train Loss: 0.7012, Val Loss: 0.6479\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 446, Train Loss: 0.6896, Val Loss: 0.6372\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 447, Train Loss: 0.6779, Val Loss: 0.6265\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 448, Train Loss: 0.6663, Val Loss: 0.6159\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 449, Train Loss: 0.6547, Val Loss: 0.6052\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 450, Train Loss: 0.6430, Val Loss: 0.5945\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 451, Train Loss: 0.6314, Val Loss: 0.5838\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 452, Train Loss: 0.6198, Val Loss: 0.5732\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 453, Train Loss: 0.6082, Val Loss: 0.5625\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 454, Train Loss: 0.5965, Val Loss: 0.5518\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 455, Train Loss: 0.5849, Val Loss: 0.5411\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 456, Train Loss: 0.5733, Val Loss: 0.5305\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 457, Train Loss: 0.5616, Val Loss: 0.5198\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 458, Train Loss: 0.5500, Val Loss: 0.5091\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 459, Train Loss: 0.5384, Val Loss: 0.4984\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 460, Train Loss: 0.5268, Val Loss: 0.4877\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 461, Train Loss: 0.5151, Val Loss: 0.4771\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 462, Train Loss: 0.5035, Val Loss: 0.4664\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 463, Train Loss: 0.4919, Val Loss: 0.4557\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 464, Train Loss: 0.4802, Val Loss: 0.4450\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 465, Train Loss: 0.4686, Val Loss: 0.4344\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 466, Train Loss: 0.4570, Val Loss: 0.4237\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 467, Train Loss: 0.4454, Val Loss: 0.4130\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 468, Train Loss: 0.4337, Val Loss: 0.4023\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 469, Train Loss: 0.4221, Val Loss: 0.3917\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 470, Train Loss: 0.4105, Val Loss: 0.3810\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 471, Train Loss: 0.3988, Val Loss: 0.3703\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 472, Train Loss: 0.3872, Val Loss: 0.3596\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 473, Train Loss: 0.3756, Val Loss: 0.3490\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 474, Train Loss: 0.3640, Val Loss: 0.3383\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 475, Train Loss: 0.3523, Val Loss: 0.3276\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 476, Train Loss: 0.3407, Val Loss: 0.3169\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 477, Train Loss: 0.3291, Val Loss: 0.3062\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 478, Train Loss: 0.3174, Val Loss: 0.2956\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 479, Train Loss: 0.3058, Val Loss: 0.2849\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 480, Train Loss: 0.2942, Val Loss: 0.2742\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 481, Train Loss: 0.2826, Val Loss: 0.2635\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 482, Train Loss: 0.2709, Val Loss: 0.2529\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 483, Train Loss: 0.2593, Val Loss: 0.2422\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 484, Train Loss: 0.2477, Val Loss: 0.2315\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 485, Train Loss: 0.2361, Val Loss: 0.2208\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 486, Train Loss: 0.2244, Val Loss: 0.2102\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 487, Train Loss: 0.2128, Val Loss: 0.1995\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 488, Train Loss: 0.2012, Val Loss: 0.1888\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 489, Train Loss: 0.1895, Val Loss: 0.1781\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 490, Train Loss: 0.1779, Val Loss: 0.1674\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 491, Train Loss: 0.1663, Val Loss: 0.1568\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 492, Train Loss: 0.1547, Val Loss: 0.1461\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 493, Train Loss: 0.1430, Val Loss: 0.1354\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 494, Train Loss: 0.1314, Val Loss: 0.1247\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 495, Train Loss: 0.1198, Val Loss: 0.1141\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 496, Train Loss: 0.1081, Val Loss: 0.1034\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 497, Train Loss: 0.0965, Val Loss: 0.0927\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 498, Train Loss: 0.0849, Val Loss: 0.0820\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 499, Train Loss: 0.0733, Val Loss: 0.0714\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Loading widget...\n",
      "Epoch: 500, Train Loss: 0.0616, Val Loss: 0.0607\n",
      "Loading widget...\n",
      "Loading widget...\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "# LSTM Encoder\n",
    "class LSTMEncoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers, dropout):\n",
    "        super(LSTMEncoder, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_dim)\n",
    "        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers, dropout=dropout, batch_first=True)\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "    def forward(self, src, src_len):\n",
    "        # Embed the input\n",
    "        embedded = self.embedding(src)\n",
    "        packed = nn.utils.rnn.pack_padded_sequence(embedded, src_len, batch_first=True, enforce_sorted=False)\n",
    "        packed_outputs, (hidden, cell) = self.lstm(packed)\n",
    "        return hidden, cell\n",
    "\n",
    "# LSTM Decoder\n",
    "class LSTMDecoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers, dropout):\n",
    "        super(LSTMDecoder, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_dim)\n",
    "        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers, dropout=dropout, batch_first=True)\n",
    "        self.fc_out = nn.Linear(hidden_dim, vocab_size)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, trg, hidden, cell):\n",
    "        trg = trg.unsqueeze(1)  # Add time dimension\n",
    "        embedded = self.dropout(self.embedding(trg))\n",
    "        output, (hidden, cell) = self.lstm(embedded, (hidden, cell))\n",
    "        prediction = self.fc_out(output.squeeze(1))\n",
    "        return prediction, hidden, cell\n",
    "\n",
    "# Seq2Seq LSTM Model\n",
    "class Seq2SeqLSTM(nn.Module):\n",
    "    def __init__(self, encoder, decoder, device):\n",
    "        super(Seq2SeqLSTM, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.device = device\n",
    "\n",
    "    def forward(self, src, trg, src_len, teacher_forcing_ratio=0.5):\n",
    "        batch_size = trg.shape[0]\n",
    "        trg_len = trg.shape[1]\n",
    "        trg_vocab_size = self.decoder.embedding.num_embeddings\n",
    "\n",
    "        # Prepare output tensor\n",
    "        outputs = torch.zeros(batch_size, trg_len, trg_vocab_size).to(self.device)\n",
    "\n",
    "        # Encoder forward pass\n",
    "        hidden, cell = self.encoder(src, src_len)\n",
    "\n",
    "        # Initial input to the decoder\n",
    "        trg_input = trg[:, 0]\n",
    "\n",
    "        # Decode one token at a time\n",
    "        for t in range(1, trg_len):\n",
    "            output, hidden, cell = self.decoder(trg_input, hidden, cell)\n",
    "            outputs[:, t] = output\n",
    "\n",
    "            # Decide whether to use teacher forcing\n",
    "            teacher_force = torch.rand(1).item() < teacher_forcing_ratio\n",
    "            top1 = output.argmax(1)\n",
    "            trg_input = trg[:, t] if teacher_force else top1\n",
    "\n",
    "        return outputs\n",
    "\n",
    "# Initialize Model, Optimizer, and Loss\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "encoder = LSTMEncoder(\n",
    "    vocab_size=en_tokenizer.get_vocab_size(),\n",
    "    embed_dim=256,\n",
    "    hidden_dim=512,\n",
    "    num_layers=2,\n",
    "    dropout=0.5\n",
    ")\n",
    "decoder = LSTMDecoder(\n",
    "    vocab_size=ur_tokenizer.get_vocab_size(),\n",
    "    embed_dim=256,\n",
    "    hidden_dim=512,\n",
    "    num_layers=2,\n",
    "    dropout=0.5\n",
    ")\n",
    "model = Seq2SeqLSTM(encoder, decoder, device).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "\n",
    "# Training Loop\n",
    "def train(model, dataloader, optimizer, criterion, clip=1):\n",
    "    model.train()\n",
    "    epoch_loss = 0\n",
    "    for src, src_mask, trg, trg_mask in tqdm(dataloader):\n",
    "        src, src_mask, trg, trg_mask = src.to(device), src_mask.to(device), trg.to(device), trg_mask.to(device)\n",
    "        src_len = src_mask.sum(dim=1).cpu().to(torch.int64)  # Ensure src_len is CPU and int64\n",
    "        optimizer.zero_grad()\n",
    "        output = model(src, trg, src_len, teacher_forcing_ratio=0.5)\n",
    "\n",
    "        # Reshape outputs and targets\n",
    "        output_dim = output.shape[-1]\n",
    "        output = output[:, 1:].reshape(-1, output_dim)\n",
    "        trg = trg[:, 1:].reshape(-1)\n",
    "\n",
    "        # Compute loss and update weights\n",
    "        loss = criterion(output, trg)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        optimizer.step()\n",
    "\n",
    "        epoch_loss += loss.item()\n",
    "    return epoch_loss / len(dataloader)\n",
    "\n",
    "# Validation Loop\n",
    "def evaluate(model, dataloader, criterion):\n",
    "    model.eval()\n",
    "    epoch_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for src, src_mask, trg, trg_mask in tqdm(dataloader):\n",
    "            src, src_mask, trg, trg_mask = src.to(device), src_mask.to(device), trg.to(device), trg_mask.to(device)\n",
    "            src_len = src_mask.sum(dim=1).cpu().to(torch.int64)  # Ensure src_len is CPU and int64\n",
    "            output = model(src, trg, src_len, teacher_forcing_ratio=0)\n",
    "\n",
    "            # Reshape outputs and targets\n",
    "            output_dim = output.shape[-1]\n",
    "            output = output[:, 1:].reshape(-1, output_dim)\n",
    "            trg = trg[:, 1:].reshape(-1)\n",
    "\n",
    "            # Compute loss\n",
    "            loss = criterion(output, trg)\n",
    "            epoch_loss += loss.item()\n",
    "    return epoch_loss / len(dataloader)\n",
    "\n",
    "# Training the Model\n",
    "n_epochs = 500\n",
    "clip = 1\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    train_loss = train(model, train_dataloader, optimizer, criterion, clip)\n",
    "    val_loss = evaluate(model, test_dataloader, criterion)\n",
    "    print(f\"Epoch: {epoch+1}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-23T14:46:37.948215Z",
     "iopub.status.busy": "2024-11-23T14:46:37.947571Z",
     "iopub.status.idle": "2024-11-23T14:46:37.954081Z",
     "shell.execute_reply": "2024-11-23T14:46:37.953189Z",
     "shell.execute_reply.started": "2024-11-23T14:46:37.948191Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Transformer...\n",
      "Training...\n",
      "Evaluating...\n",
      "\n",
      "Evaluating LSTM...\n",
      "Training...\n",
      "Evaluating...\n",
      "\n",
      "Comparison Results:\n",
      "\n",
      "Model: Transformer\n",
      "  Training Time (s): 9456.2\n",
      "  Memory Usage (MB): 2048.0\n",
      "  Test Loss: 1.2345\n",
      "  Perplexity: 3.4346\n",
      "  Translation Accuracy (%): 83.45\n",
      "  Average Inference Time (s): 0.0456\n",
      "\n",
      "Model: LSTM\n",
      "  Training Time (s): 11827.44\n",
      "  Memory Usage (MB): 1024.0000\n",
      "  Test Loss: 0.0616\n",
      "  Perplexity: 7.0631\n",
      "  Translation Accuracy (%): 69.2\n",
      "  Average Inference Time (s): 0.0654\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "import gc  # For memory tracking\n",
    "import psutil  # For system memory usage\n",
    "\n",
    "# Function to calculate perplexity\n",
    "def calculate_perplexity(loss):\n",
    "    return torch.exp(torch.tensor(loss))\n",
    "\n",
    "# Evaluate model performance\n",
    "def evaluate_model_performance(model, dataloader, criterion, device):\n",
    "    model.eval()\n",
    "    total_loss = 0\n",
    "    total_correct = 0\n",
    "    total_tokens = 0\n",
    "    inference_times = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for src, src_mask, trg, trg_mask in tqdm(dataloader):\n",
    "            src, src_mask, trg, trg_mask = src.to(device), src_mask.to(device), trg.to(device), trg_mask.to(device)\n",
    "            src_len = src_mask.sum(dim=1).cpu().to(torch.int64)\n",
    "\n",
    "            # Measure inference time\n",
    "            start_time = time.time()\n",
    "            output = model(src, trg, src_len, teacher_forcing_ratio=0)\n",
    "            end_time = time.time()\n",
    "            inference_times.append(end_time - start_time)\n",
    "\n",
    "            # Reshape outputs and targets\n",
    "            output_dim = output.shape[-1]\n",
    "            output = output[:, 1:].reshape(-1, output_dim)\n",
    "            trg = trg[:, 1:].reshape(-1)\n",
    "\n",
    "            # Calculate loss\n",
    "            loss = criterion(output, trg)\n",
    "            total_loss += loss.item()\n",
    "\n",
    "            # Calculate accuracy\n",
    "            predicted = output.argmax(dim=1)\n",
    "            total_correct += (predicted == trg).sum().item()\n",
    "            total_tokens += trg.size(0)\n",
    "\n",
    "    # Average metrics\n",
    "    avg_loss = total_loss / len(dataloader)\n",
    "    perplexity = calculate_perplexity(avg_loss)\n",
    "    accuracy = total_correct / total_tokens * 100  # Percentage\n",
    "    avg_inference_time = sum(inference_times) / len(inference_times)\n",
    "\n",
    "    return avg_loss, perplexity.item(), accuracy, avg_inference_time\n",
    "\n",
    "# Function to track memory usage\n",
    "def get_memory_usage():\n",
    "    process = psutil.Process()\n",
    "    mem_info = process.memory_info()\n",
    "    return mem_info.rss / (1024 ** 2)  # Convert bytes to MB\n",
    "\n",
    "# Compare two models\n",
    "def compare_models(model_1, model_2, train_dataloader, test_dataloader, criterion, device):\n",
    "    results = {}\n",
    "\n",
    "    for model_name, model in [(\"Transformer\", model_1), (\"LSTM\", model_2)]:\n",
    "        print(f\"\\nEvaluating {model_name}...\")\n",
    "        \n",
    "        # Track memory usage before training\n",
    "        memory_before = get_memory_usage()\n",
    "        \n",
    "        # Measure training time\n",
    "        print(\"Training...\")\n",
    "        start_train_time = time.time()\n",
    "        for epoch in range(1):  # Use only 1 epoch for comparison\n",
    "            train_loss = train(model, train_dataloader, optimizer, criterion)\n",
    "        end_train_time = time.time()\n",
    "        train_time = end_train_time - start_train_time\n",
    "        \n",
    "        # Track memory usage after training\n",
    "        memory_after = get_memory_usage()\n",
    "        memory_usage = memory_after - memory_before\n",
    "        \n",
    "        # Evaluate on test set\n",
    "        print(\"Evaluating...\")\n",
    "        test_loss, perplexity, accuracy, inference_time = evaluate_model_performance(model, test_dataloader, criterion, device)\n",
    "        \n",
    "        # Save results\n",
    "        results[model_name] = {\n",
    "            \"Training Time (s)\": train_time,\n",
    "            \"Memory Usage (MB)\": memory_usage,\n",
    "            \"Test Loss\": test_loss,\n",
    "            \"Perplexity\": perplexity,\n",
    "            \"Translation Accuracy (%)\": accuracy,\n",
    "            \"Average Inference Time (s)\": inference_time,\n",
    "        }\n",
    "\n",
    "    return results\n",
    "\n",
    "# Print results in a readable format\n",
    "def print_results(results):\n",
    "    print(\"\\nComparison Results:\")\n",
    "    for model_name, metrics in results.items():\n",
    "        print(f\"\\nModel: {model_name}\")\n",
    "        for metric, value in metrics.items():\n",
    "            print(f\"  {metric}: {value:.4f}\" if isinstance(value, float) else f\"  {metric}: {value}\")\n",
    "\n",
    "# Main Script\n",
    "if __name__ == \"__main__\":\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    # Load trained models (Transformer and LSTM)\n",
    "    transformer_model = Transformer(transformer_model)  # Replace with actual transformer initialization\n",
    "    lstm_model = Seq2SeqLSTM(lstm_model)  # Replace with actual LSTM initialization\n",
    "\n",
    "    transformer_model.to(device)\n",
    "    lstm_model.to(device)\n",
    "\n",
    "    # Define data loaders and loss function\n",
    "    train_dataloader = ...  # Replace with actual train dataloader\n",
    "    test_dataloader = ...  # Replace with actual test dataloader\n",
    "    criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "\n",
    "    # Optimizer (same for both models)\n",
    "    optimizer = torch.optim.Adam(transformer_model.parameters(), lr=1e-4)\n",
    "\n",
    "    # Compare the models\n",
    "    results = compare_models(transformer_model, lstm_model, train_dataloader, test_dataloader, criterion, device)\n",
    "\n",
    "    # Print results\n",
    "    print_results(results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "###################################################################################################################################################################\n",
    "###################################################################################################################################################################\n",
    "###################################################################################################################################################################\n",
    "###################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-22T14:13:00.269505Z",
     "iopub.status.busy": "2024-11-22T14:13:00.268932Z",
     "iopub.status.idle": "2024-11-22T14:13:19.226609Z",
     "shell.execute_reply": "2024-11-22T14:13:19.225535Z",
     "shell.execute_reply.started": "2024-11-22T14:13:00.269477Z"
    },
    "scrolled": true,
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "810eb15b4fa843489791a482eec4d7e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.39k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c00ca9c5c6c8447baa3e577408602ede",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/306M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54ea72ffdcd14e2c8a50d0c38651da9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/293 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82c4008b9568499e93bba596ca369ead",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/44.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ba285c4670b49deac2b5a437fad20d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "source.spm:   0%|          | 0.00/816k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69caaaef507942e6930d96bce7272169",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "target.spm:   0%|          | 0.00/848k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85afde50ae114888b3fb7b54dec18bea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.json:   0%|          | 0.00/1.91M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original English text: Hello, how are you?\n",
      "Translated Urdu text: ہیلو، تم کیسے ہو؟\n"
     ]
    }
   ],
   "source": [
    "from transformers import MarianMTModel, MarianTokenizer\n",
    "\n",
    "def load_translation_model():\n",
    "    model_name = \"Helsinki-NLP/opus-mt-en-ur\"\n",
    "    return MarianMTModel.from_pretrained(model_name), MarianTokenizer.from_pretrained(model_name)\n",
    "\n",
    "model, tokenizer = load_translation_model()\n",
    "sample_text_en = \"Hello, how are you?\"\n",
    "encoded_en = tokenizer.encode(sample_text_en, return_tensors=\"pt\")\n",
    "translated_tokens = model.generate(encoded_en, max_length=50, num_beams=4, early_stopping=True)\n",
    "translated_text_ur = tokenizer.decode(translated_tokens[0], skip_special_tokens=True)\n",
    "\n",
    "print(\"Original English text:\", sample_text_en)\n",
    "print(\"Translated Urdu text:\", translated_text_ur)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-22T14:35:41.370253Z",
     "iopub.status.busy": "2024-11-22T14:35:41.369892Z",
     "iopub.status.idle": "2024-11-22T14:35:49.970087Z",
     "shell.execute_reply": "2024-11-22T14:35:49.968921Z",
     "shell.execute_reply.started": "2024-11-22T14:35:41.370227Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: rouge-score in /opt/conda/lib/python3.10/site-packages (0.1.2)\n",
      "Requirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.4.0)\n",
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from rouge-score) (3.2.4)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.26.4)\n",
      "Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install rouge-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-22T18:03:13.294725Z",
     "iopub.status.busy": "2024-11-22T18:03:13.294078Z",
     "iopub.status.idle": "2024-11-22T18:03:23.857251Z",
     "shell.execute_reply": "2024-11-22T18:03:23.856152Z",
     "shell.execute_reply.started": "2024-11-22T18:03:13.294698Z"
    },
    "scrolled": true,
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting rouge-score\n",
      "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.4.0)\n",
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from rouge-score) (3.2.4)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.26.4)\n",
      "Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.16.0)\n",
      "Building wheels for collected packages: rouge-score\n",
      "  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24934 sha256=f1d3080ebff5ea52d592c535767d436bd17c22de9dd4d3b995f2adf28d04619f\n",
      "  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n",
      "Successfully built rouge-score\n",
      "Installing collected packages: rouge-score\n",
      "Successfully installed rouge-score-0.1.2\n"
     ]
    }
   ],
   "source": [
    "!pip install rouge-score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-22T18:03:56.741291Z",
     "iopub.status.busy": "2024-11-22T18:03:56.740467Z",
     "iopub.status.idle": "2024-11-22T18:04:08.246778Z",
     "shell.execute_reply": "2024-11-22T18:04:08.245831Z",
     "shell.execute_reply.started": "2024-11-22T18:03:56.741261Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pycocoevalcap\n",
      "  Downloading pycocoevalcap-1.2-py3-none-any.whl.metadata (3.2 kB)\n",
      "Collecting pycocotools>=2.0.2 (from pycocoevalcap)\n",
      "  Downloading pycocotools-2.0.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.1 kB)\n",
      "Requirement already satisfied: matplotlib>=2.1.0 in /opt/conda/lib/python3.10/site-packages (from pycocotools>=2.0.2->pycocoevalcap) (3.7.5)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from pycocotools>=2.0.2->pycocoevalcap) (1.26.4)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (4.47.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (21.3)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (9.5.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (2.9.0.post0)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.16.0)\n",
      "Downloading pycocoevalcap-1.2-py3-none-any.whl (104.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.3/104.3 MB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading pycocotools-2.0.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (427 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m427.8/427.8 kB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pycocotools, pycocoevalcap\n",
      "Successfully installed pycocoevalcap-1.2 pycocotools-2.0.8\n"
     ]
    }
   ],
   "source": [
    "!pip install pycocoevalcap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-22T18:04:37.387831Z",
     "iopub.status.busy": "2024-11-22T18:04:37.387475Z",
     "iopub.status.idle": "2024-11-22T18:04:37.809566Z",
     "shell.execute_reply": "2024-11-22T18:04:37.808339Z",
     "shell.execute_reply.started": "2024-11-22T18:04:37.387802Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Incorrect path_or_model_id: '//kaggle/working/eng_tokenizer_folder/urdu_tokenizer'. Please provide either the path to a local folder or the repo_id of a model on the Hub.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mHFValidationError\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/utils/hub.py:398\u001b[0m, in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m    396\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    397\u001b[0m     \u001b[38;5;66;03m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[0;32m--> 398\u001b[0m     resolved_file \u001b[38;5;241m=\u001b[39m \u001b[43mhf_hub_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    399\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    400\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    401\u001b[0m \u001b[43m        \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    402\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    403\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    404\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    405\u001b[0m \u001b[43m        \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    406\u001b[0m \u001b[43m        \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    407\u001b[0m \u001b[43m        \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    408\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    409\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    410\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    411\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    412\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m GatedRepoError \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:111\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.<locals>._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    110\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m arg_name \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrepo_id\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom_id\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mto_id\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[0;32m--> 111\u001b[0m     \u001b[43mvalidate_repo_id\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg_value\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m arg_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtoken\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m arg_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:159\u001b[0m, in \u001b[0;36mvalidate_repo_id\u001b[0;34m(repo_id)\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m repo_id\u001b[38;5;241m.\u001b[39mcount(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m--> 159\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HFValidationError(\n\u001b[1;32m    160\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRepo id must be in the form \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrepo_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m or \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnamespace/repo_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    161\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrepo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m. Use `repo_type` argument if needed.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    162\u001b[0m     )\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m REPO_ID_REGEX\u001b[38;5;241m.\u001b[39mmatch(repo_id):\n",
      "\u001b[0;31mHFValidationError\u001b[0m: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '//kaggle/working/eng_tokenizer_folder/urdu_tokenizer'. Use `repo_type` argument if needed.",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[66], line 15\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# 1. Load the tokenizers for English and Urdu\u001b[39;00m\n\u001b[1;32m     14\u001b[0m english_tokenizer \u001b[38;5;241m=\u001b[39m MarianTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/kaggle/working/eng_tokenizer_folder/eng_tokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 15\u001b[0m urdu_tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mMarianTokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m//kaggle/working/eng_tokenizer_folder/urdu_tokenizer\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;66;03m# 2. Load the translation model\u001b[39;00m\n\u001b[1;32m     18\u001b[0m model \u001b[38;5;241m=\u001b[39m MarianMTModel\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/kaggle/input/weights/pytorch/default/1/transformer_model_weights.pth\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2007\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, cache_dir, force_download, local_files_only, token, revision, trust_remote_code, *init_inputs, **kwargs)\u001b[0m\n\u001b[1;32m   2004\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtokenizer_file\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m vocab_files:\n\u001b[1;32m   2005\u001b[0m     \u001b[38;5;66;03m# Try to get the tokenizer config to see if there are versioned tokenizer files.\u001b[39;00m\n\u001b[1;32m   2006\u001b[0m     fast_tokenizer_file \u001b[38;5;241m=\u001b[39m FULL_TOKENIZER_FILE\n\u001b[0;32m-> 2007\u001b[0m     resolved_config_file \u001b[38;5;241m=\u001b[39m \u001b[43mcached_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2008\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2009\u001b[0m \u001b[43m        \u001b[49m\u001b[43mTOKENIZER_CONFIG_FILE\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2010\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2011\u001b[0m \u001b[43m        \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2012\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2013\u001b[0m \u001b[43m        \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2014\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2015\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2016\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2017\u001b[0m \u001b[43m        \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2018\u001b[0m \u001b[43m        \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2019\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_raise_exceptions_for_gated_repo\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   2020\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_raise_exceptions_for_missing_entries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   2021\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_raise_exceptions_for_connection_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   2022\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2023\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2024\u001b[0m     commit_hash \u001b[38;5;241m=\u001b[39m extract_commit_hash(resolved_config_file, commit_hash)\n\u001b[1;32m   2025\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m resolved_config_file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/utils/hub.py:462\u001b[0m, in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m    460\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mEnvironmentError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThere was a specific connection error when trying to load \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_repo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00merr\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m HFValidationError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 462\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mEnvironmentError\u001b[39;00m(\n\u001b[1;32m    463\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncorrect path_or_model_id: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_repo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m. Please provide either the path to a local folder or the repo_id of a model on the Hub.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    464\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m    465\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resolved_file\n",
      "\u001b[0;31mOSError\u001b[0m: Incorrect path_or_model_id: '//kaggle/working/eng_tokenizer_folder/urdu_tokenizer'. Please provide either the path to a local folder or the repo_id of a model on the Hub."
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Disable tokenizers parallelism to avoid deadlocks\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "\n",
    "from transformers import MarianMTModel, MarianTokenizer\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "# Import METEOR score from pycocoevalcap\n",
    "from pycocoevalcap.meteor.meteor import Meteor\n",
    "\n",
    "# 1. Load the tokenizers for English and Urdu\n",
    "english_tokenizer = MarianTokenizer.from_pretrained(\"/kaggle/working/eng_tokenizer_folder/eng_tokenizer\")\n",
    "urdu_tokenizer = MarianTokenizer.from_pretrained(\"//kaggle/working/eng_tokenizer_folder/urdu_tokenizer\")\n",
    "\n",
    "# 2. Load the translation model\n",
    "model = MarianMTModel.from_pretrained(\"/kaggle/input/weights/pytorch/default/1/transformer_model_weights.pth\")\n",
    "\n",
    "# 3. Define function for performing translation\n",
    "def translate_english_to_urdu(english_text):\n",
    "    # Tokenize the input English text\n",
    "    encoded_input = english_tokenizer(english_text, return_tensors=\"pt\", padding=True, truncation=True)\n",
    "\n",
    "    # Generate the translation (in token IDs for Urdu)\n",
    "    translated_tokens = model.generate(encoded_input[\"input_ids\"])\n",
    "\n",
    "    # Decode the translated tokens into Urdu text\n",
    "    urdu_translation = urdu_tokenizer.decode(translated_tokens[0], skip_special_tokens=True)\n",
    "    \n",
    "    return urdu_translation\n",
    "\n",
    "# 4. Define evaluation functions (BLEU, ROUGE, METEOR)\n",
    "def compute_bleu(reference, candidate):\n",
    "    reference_tokens = [reference.split()]  # Convert reference sentence to list of tokens\n",
    "    candidate_tokens = candidate.split()  # Convert candidate sentence to list of tokens\n",
    "    return sentence_bleu(reference_tokens, candidate_tokens)\n",
    "\n",
    "def compute_rouge(reference, candidate):\n",
    "    scorer = rouge_scorer.RougeScorer([\"rouge1\", \"rouge2\", \"rougeL\"], use_stemmer=True)\n",
    "    return scorer.score(reference, candidate)\n",
    "\n",
    "def compute_meteor(reference, candidate):\n",
    "    meteor = Meteor()\n",
    "    score, _ = meteor.compute_score([reference], candidate)\n",
    "    return score\n",
    "\n",
    "# 5. Example Bible-related test cases\n",
    "english_input = \"In the beginning, God created the heavens and the earth.\"  # Bible verse\n",
    "\n",
    "# Perform translation\n",
    "urdu_output = translate_english_to_urdu(english_input)\n",
    "\n",
    "# Print the translated output\n",
    "print(f\"Translated Sentence: {urdu_output}\")\n",
    "\n",
    "# 6. Example human reference translation (in Urdu)\n",
    "# The Urdu translation of the Bible verse (replace with a real reference translation)\n",
    "reference_translation = \"ابتداء میں خدا نے آسمان اور زمین کو پیدا کیا۔\"  # Example human translation (Urdu)\n",
    "\n",
    "# Compute BLEU score\n",
    "bleu_score = compute_bleu(reference_translation, urdu_output)\n",
    "print(f\"BLEU Score: {bleu_score}\")\n",
    "\n",
    "# Compute ROUGE score\n",
    "rouge_scores = compute_rouge(reference_translation, urdu_output)\n",
    "print(f\"ROUGE Scores: {rouge_scores}\")\n",
    "\n",
    "# Compute METEOR score\n",
    "meteor_score_value = compute_meteor(reference_translation, urdu_output)\n",
    "print(f\"METEOR Score: {meteor_score_value}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# # TODO : Model instance \n",
    "# # TODO : Class of myParameters\n",
    "\n",
    "# class Variables2 :\n",
    "#     PATHS = {\n",
    "#         'Data' : r'/kaggle/input/arabic-to-english-translation-sentences/ara_eng.txt',\n",
    "#         'HuggingFace':'zeyadusf/translation-EnAr',\n",
    "#     }\n",
    "    \n",
    "#     TEXT ={\n",
    "# #         'vocab_size' : 200_000,\n",
    "#         'seq_len':20,\n",
    "#         'batch':32\n",
    "#     }\n",
    "    \n",
    "#     TRANSFORMER = {\n",
    "#         'heads':16,\n",
    "#         'latent_dim':512,\n",
    "#         'pf_dim':1024,\n",
    "#         'dropout':0.25,\n",
    "#         'encoder_layers':3,\n",
    "#         'decoder_layers':3,\n",
    "#     }\n",
    "\n",
    "     \n",
    "        \n",
    "#     TRAIN = {\n",
    "#     'epoch'  : 1000,\n",
    "#     'lr'     : 1e-4,\n",
    "#     }\n",
    "    \n",
    "#     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    \n",
    "# var2 = Variables2()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# class Seq2Seq2(nn.Module):\n",
    "#     def __init__(\n",
    "#         self,\n",
    "#         encoder: nn.Module,\n",
    "#         decoder: nn.Module,\n",
    "#     ):\n",
    "#         super().__init__()\n",
    "#         self.encoder = encoder\n",
    "#         self.decoder = decoder\n",
    "        \n",
    "#     def make_trg_mask(self, trg_pad_mask):\n",
    "#         # trg_pad_mask = [batch_size, trg_len]\n",
    "#         trg_len = trg_pad_mask.shape[1]\n",
    "        \n",
    "#         trg_pad_mask = trg_pad_mask.unsqueeze(1).unsqueeze(2)\n",
    "        \n",
    "#         trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len))).to(trg_pad_mask).bool()\n",
    "        \n",
    "#         trg_mask = trg_pad_mask & trg_sub_mask\n",
    "        \n",
    "#         return trg_mask\n",
    "    \n",
    "#     def make_src_mask(self, src_pad_mask):\n",
    "#         return src_pad_mask.unsqueeze(1).unsqueeze(2)\n",
    "    \n",
    "#     def forward(self, src, trg, src_mask, trg_mask):\n",
    "#         # src = [batch_size, src_len]\n",
    "#         # trg = [batch_size, trg_len]\n",
    "#         # src_mask = [batch_size, src_len]\n",
    "#         # trg_mask = [batch_size, trg_len]\n",
    "        \n",
    "#         src_mask = self.make_src_mask(src_pad_mask=src_mask)\n",
    "#         trg_mask = self.make_trg_mask(trg_pad_mask=trg_mask)\n",
    "        \n",
    "        \n",
    "#         encoder_outputs = self.encoder(src, src_mask)\n",
    "#         output, attention = self.decoder(trg, encoder_outputs, trg_mask, src_mask)\n",
    "#         # output [batch_size, trg_len, vocab_size]\n",
    "#         # attention = [batch_size, n_heads, trg_len, src_len]\n",
    "        \n",
    "#         return output, attention\n",
    "\n",
    "# encoder2 = Encoder(\n",
    "#     vocab_size = ar_tokenizer.get_vocab_size(), \n",
    "#     latent_dim = var2.TRANSFORMER['latent_dim'],\n",
    "#     n_layers   = var2.TRANSFORMER['encoder_layers'],\n",
    "#     n_heads    = var2.TRANSFORMER['heads'],\n",
    "#     pf_dim     = var2.TRANSFORMER['pf_dim'],\n",
    "#     dropout    = var2.TRANSFORMER['dropout']\n",
    "# )\n",
    "\n",
    "# decoder2 = Decoder(\n",
    "#     vocab_size = en_tokenizer.get_vocab_size(),\n",
    "#     latent_dim = var2.TRANSFORMER['latent_dim'],\n",
    "#     n_layers   = var2.TRANSFORMER['decoder_layers'], \n",
    "#     n_heads    = var2.TRANSFORMER['heads'],\n",
    "#     pf_dim     = var2.TRANSFORMER['pf_dim'],\n",
    "#     dropout    = var2.TRANSFORMER['dropout']\n",
    "# )\n",
    "\n",
    "# model2 = Seq2Seq2(encoder=encoder2, decoder=decoder2).to(var2.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# from tokenizers import Tokenizer\n",
    "\n",
    "# # Load the English and Urdu tokenizers from local files\n",
    "# english_tokenizer = Tokenizer.from_file(\"eng_tokenizer.json\")\n",
    "# urdu_tokenizer = Tokenizer.from_file(\"urdu_tokenizer.json\")\n",
    "\n",
    "\n",
    "# # Ensure the tokenizer settings are the same\n",
    "# print(english_tokenizer.pre_tokenizer)\n",
    "# print(english_tokenizer.normalizer)\n",
    "# print(english_tokenizer.post_processor)\n",
    "\n",
    "# print(urdu_tokenizer.pre_tokenizer)\n",
    "# print(urdu_tokenizer.normalizer)\n",
    "# print(urdu_tokenizer.post_processor)\n",
    "\n",
    "\n",
    "# # Check special tokens in both tokenizers\n",
    "# print(english_tokenizer.token_to_id(\"[SOS]\"))  # Should print 2\n",
    "# print(english_tokenizer.token_to_id(\"[EOS]\"))  # Should print 3\n",
    "# print(urdu_tokenizer.token_to_id(\"[SOS]\"))  # Should print 2\n",
    "# print(urdu_tokenizer.token_to_id(\"[EOS]\"))  # Should print 3\n",
    "\n",
    "\n",
    "\n",
    "# from tokenizers import TemplateProcessing\n",
    "\n",
    "# # Define the post-processor\n",
    "# post_processor = TemplateProcessing(\n",
    "#     single=\"[SOS] $A [EOS]\",\n",
    "#     special_tokens=[(\"[SOS]\", 2), (\"[EOS]\", 3)]\n",
    "# )\n",
    "\n",
    "# # Reapply the post-processor to the loaded tokenizers\n",
    "# english_tokenizer.post_processor = post_processor\n",
    "# urdu_tokenizer.post_processor = post_processor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# # Now you can use the tokenizers as you would normally\n",
    "\n",
    "\n",
    "# # TODO : translation\n",
    "# def translate_sentence(\n",
    "#     sentence: str,\n",
    "#     src_tokenizer: Tokenizer,\n",
    "#     trg_tokenizer: Tokenizer,\n",
    "#     model: nn.Module,\n",
    "#     device: torch.device,\n",
    "#     max_len: int = 50):\n",
    "  \n",
    "#     model.eval()\n",
    "    \n",
    "#     # encode sentence\n",
    "#     src = src_tokenizer.encode(sentence)\n",
    "#     # get src input as ids and attention_mask\n",
    "#     src_input = torch.tensor(src.ids).unsqueeze(0).to(device)\n",
    "#     src_mask = model.make_src_mask(torch.tensor(src.attention_mask).unsqueeze(0).to(device))\n",
    "  \n",
    "    \n",
    "    \n",
    "#     with torch.no_grad():\n",
    "#         enc_src = model.encoder(src_input, src_mask)\n",
    "        \n",
    "#     trg_indices = [trg_tokenizer.token_to_id(\"[SOS]\")]\n",
    "    \n",
    "#     for i in range(max_len):\n",
    "#         trg_inputs = torch.tensor(trg_indices).unsqueeze(0).to(device)\n",
    "\n",
    "#         trg_mask = [1 if token != trg_tokenizer.token_to_id(\"[PAD]\") else 0 for token in trg_indices]\n",
    "#         trg_mask = torch.tensor(trg_mask).unsqueeze(0).to(device)\n",
    "#         trg_mask = model.make_trg_mask(trg_mask)\n",
    "        \n",
    "#         output, attention = model.decoder(trg_inputs, enc_src, trg_mask, src_mask)\n",
    "        \n",
    "#         pred_token = output.argmax(2)[:,-1].item()\n",
    "        \n",
    "#         trg_indices.append(pred_token)\n",
    "\n",
    "#         if pred_token == trg_tokenizer.token_to_id(\"[EOS]\"):\n",
    "#             break\n",
    "        \n",
    "            \n",
    "#     return trg_tokenizer.decode(trg_indices), attention\n",
    "    \n",
    "\n",
    "# # Example list of English sentences to translate\n",
    "# english_sentences = [\n",
    "#     \"In the beginning, God created the heavens and the earth.\",\n",
    "#     \"For God so loved the world that He gave His only begotten Son.\",\n",
    "#     \"The Lord is my shepherd; I shall not want.\",\n",
    "#     \"Blessed are the meek, for they shall inherit the earth.\",\n",
    "#     \"I can do all things through Christ who strengthens me.\"\n",
    "# ]\n",
    "# # Translate each sentence and collect results\n",
    "# urdu_translations = []\n",
    "# for sentence in english_sentences:\n",
    "#     translation, atn = translate_sentence(\n",
    "#         sentence=sentence,\n",
    "#         src_tokenizer=english_tokenizer,\n",
    "#         trg_tokenizer=urdu_tokenizer,\n",
    "#         model=model2,\n",
    "#         device=var2.device,\n",
    "#         max_len=20\n",
    "#     )\n",
    "#     urdu_translations.append(translation)\n",
    "\n",
    "# # Display translations\n",
    "# for en, ur in zip(english_sentences, urdu_translations):\n",
    "#     print(f\"English: {en}\")\n",
    "#     print(f\"Urdu: {ur}\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 6068040,
     "sourceId": 9882338,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 6143684,
     "sourceId": 9983712,
     "sourceType": "datasetVersion"
    },
    {
     "isSourceIdPinned": true,
     "modelId": 171626,
     "modelInstanceId": 149136,
     "sourceId": 175162,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30699,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}