E_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "68353d11",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus.reader.plaintext import PlaintextCorpusReader\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "75852b36",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpuspath = 'D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\unclean_text_data_test\\\\'\n",
    "corpus = PlaintextCorpusReader(corpuspath, '.*' , )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d817674b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcce8cae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a465e246",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4cd2daa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def normalize_corpus(corpus_passed):\n",
    "    counter = 0\n",
    "    document_keys = {}\n",
    "    normalized_corpus = []    \n",
    "    for fileid in corpus_passed.fileids():\n",
    "        text = corpus_passed.raw(fileid)\n",
    "        tokenize_text = corpus_passed.words(fileid)\n",
    "        filtered_tokens = []\n",
    "        filtered_text = ' '.join(filtered_tokens)\n",
    "        # Saving Key for document\n",
    "        \n",
    "        #text_pattern = r'[a-zA-Z]'\n",
    "        #cleantext2 = re.sub(r'[a-zA-Z]', '', text)\n",
    "        \n",
    "        #print(cleantext2)\n",
    "        normalized_corpus.append(filtered_text)\n",
    "        document_keys[counter] =  fileid\n",
    "        counter = counter + 1\n",
    "        #print('__________________________________________')\n",
    "    return normalized_corpus , document_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edbd47b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "Sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "07ed838d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "# from gensim.test.utils import common_texts, get_tmpfile\n",
    "#from gensim.models import Word2Vec\n",
    "# We improve our feature matrix builder with 3 additional optional parameters\n",
    "# This allows us to extract not only word features, but also n-gram features\n",
    "# We can also set the minimum and maximum frequencies to be considered as valid\n",
    "# NB: All these are simply passed on to sklearn's Vectorizer classes\n",
    "def build_feature_matrix(documents, feature_type='frequency', ngram_range=(1, 1), min_df=0.01, max_df=0.85):\n",
    "    # max_df = 0.50 means \"ignore terms that appear in more than 50% of the documents\".\n",
    "    # max_df = 25 means \"ignore terms that appear in more than 25 documents\".\n",
    "    # default max_df is 1.0, which means \"ignore terms that appear in more than 100% of the documents\". \n",
    "    # Thus, the default setting does not ignore any terms.\n",
    "    \n",
    "    # min_df = 0.01 means \"ignore terms that appear in less than 1% of the documents\".\n",
    "    # min_df = 5 means \"ignore terms that appear in less than 5 documents\".\n",
    "    # default min_df is 1, which means \"ignore terms that appear in less than 1 document\". \n",
    "    # Thus, the default setting does not ignore any terms.\n",
    "    \n",
    "    # We are setting max df to 85%\n",
    "\n",
    "    feature_type = feature_type.lower().strip()  \n",
    "    \n",
    "    if feature_type == 'binary':\n",
    "        vectorizer = CountVectorizer(binary=True, min_df=min_df,\n",
    "                                     max_df=max_df, ngram_range=ngram_range)\n",
    "    elif feature_type == 'frequency':\n",
    "        vectorizer = CountVectorizer(binary=False, min_df=min_df,\n",
    "                                     max_df=max_df, ngram_range=ngram_range)\n",
    "    elif feature_type == 'tfidf':\n",
    "        vectorizer = TfidfVectorizer( ngram_range=ngram_range)\n",
    "    elif feature_type =='word2vec' :\n",
    "        vectorizer = TfidfVectorizer(min_df=min_df, max_df=max_df, \n",
    "                                 ngram_range=ngram_range)\n",
    "    else:\n",
    "        raise Exception(\"Wrong feature type entered. Possible values: 'binary', 'frequency', 'tfidf'\")\n",
    "\n",
    "    feature_matrix = vectorizer.fit_transform(documents).astype(float)\n",
    "    \n",
    "    return vectorizer, feature_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "d97dd4fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# build_feature_matrix(['I have a dream', 'go to hell'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a0fc9f14",
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_corpus, row_id_keys = normalize_corpus(corpus)\n",
    "\n",
    "# Extract (tf-idf) features using out utils library\n",
    "tfidf_vectorizer, tfidf_feature_matrix = build_feature_matrix(normalized_corpus,\n",
    "                                                  feature_type='tfidf',\n",
    "                                                  #min_df=0.01, \n",
    "                                                  max_df=0.85,\n",
    "                                                  ngram_range=(1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4ad11814",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_doc = ['loving FUck this blue sky today']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57caaf20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e9126afb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['blue', 'fuck', 'loving', 'sky', 'this', 'today']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7b7faa4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66f2d78e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fab60030",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6bb1358d",
   "metadata": {},
   "outputs": [],
   "source": [
    "binary_vectorizer, binary_feature_matrix = build_feature_matrix(normalized_corpus,\n",
    "                                                  feature_type='binary',\n",
    "                                                  #min_df=0.01, \n",
    "                                                  max_df=0.85,\n",
    "                                                  ngram_range=(1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "af24421d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_feature_matrix = tfidf_feature_matrix[:40]\n",
    "test_feature_matrix = tfidf_feature_matrix[40:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f1914294",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "\n",
    "# Define the k-means clustering function which will call sklearn's KMeans function\n",
    "# with our parameters\n",
    "def k_means(feature_matrix, num_clusters):\n",
    "    km = KMeans(n_clusters=num_clusters,\n",
    "                max_iter=10000)\n",
    "    km.fit(feature_matrix)\n",
    "    clusters = km.labels_\n",
    "    return km, clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f79b3c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e0f6d02b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3B9wLEPpb4.11331682998371562.txt'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus.fileids()[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ee6701d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTNT\n",
    "# corpus.fileids()[5]\n",
    "# # '5ZXOjh7b8b.051154254863466364.txt'\n",
    "# txt = corpus.raw('5ZXOjh7b8b.051154254863466364.txt')\n",
    "# vectorized=tfidf_vectorizer.fit_transform(corpus.words('5ZXOjh7b8b.051154254863466364.txt'))\n",
    "# num_samples, num_features=vectorized.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "bacc0d2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61, 4188)\n",
      "['0d9lb41kwa', '10m', '10th', '11bq3eiy6i', '11th', '12th', '13th', '14th', '150th', '15th', '16th', '17th', '18th', '1970s', '1h', '1m', '1st', '20m', '21st', '26th', '2h', '2nd', '2sinrgotuf', '3ice', '3m', '3rd', '3s', '55m', '57m', '5m', '60s', '70s', '75m', '76ers', '7t0p0r8pyr', '83e6vwhwop', '8m', '8u7cea6fm1', '90s', '9gem', '9m', 'aa', 'abc', 'abe', 'abide', 'aboard', 'about', 'above', 'absence', 'absorbed']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "print(tfidf_feature_matrix.shape)     \n",
    "\n",
    "# Get the names of the features\n",
    "feature_names = tfidf_vectorizer.get_feature_names()\n",
    "\n",
    "# Show the first twenty features (check more if needed)\n",
    "print(feature_names[:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "213cd8af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d736cf0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "979f4ad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "\n",
    "# Define the k-means clustering function which will call sklearn's KMeans function\n",
    "# with our parameters\n",
    "def k_means(feature_matrix, num_clusters=5):\n",
    "    km = KMeans(n_clusters=num_clusters,\n",
    "                max_iter=10000)\n",
    "    km.fit(feature_matrix)\n",
    "    clusters = km.labels_\n",
    "    return km, clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7a48d85a",
   "metadata": {},
   "outputs": [],
   "source": [
    "distortions = []\n",
    "K = range(1,10)\n",
    "# Running K-Means with a range of k\n",
    "for k in K: \n",
    "    km_obj, clusters = k_means(feature_matrix=train_feature_matrix,\n",
    "                           num_clusters=k)\n",
    "    # inertia_ is the distance from each sample to the centroid.\n",
    "    distortions.append(km_obj.inertia_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "761c558a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6wAAAHwCAYAAACi6OLhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABILElEQVR4nO3dd5iU1d3/8feXroAVUCyIIBbUqBGiSdSIPUZjNJFojDGaPBpL7KLwxOdJtRA1RpMYNRo1VtRojPpYYrA3wArYEEUsUewUkXZ+f5zZ3zbKAjt7z+y+X9d1X7tzz+zMZ3ZI5MO5zzmRUkKSJEmSpErTrugAkiRJkiQtjIVVkiRJklSRLKySJEmSpIpkYZUkSZIkVSQLqyRJkiSpIllYJUmSJEkVycIqSa1ARPw8Iq5ugdfpGxEpIjqUbt8fET8u9+u2hOZ8LxFxRUT8ehl+LkXEBs2RYRHPv31EvFSu51/I65X1/SyriBgREX8p03O/HhG7LOK+ZfpzIUltmYVVkqpARMyocyyIiM/q3D6omV/rioiY0+A1n23O11hWdQrzUw3O9yhlfr2Jz9MiBb/SpJQeSiltVI7nrtR/vIiIHSPizbrnUkpnpJQqLqskqTELqyRVgZRSt5oDeAPYu865a8rwkiPrvmZKaYsyvMby6BoRm9W5/T3gtaLCSJKk8rCwSlLr0SkiroqI6RExISIG1dwREWtFxM0RMS0iXouIY5vxdftHxJMR8UlE/CMiVqvzut8sZfm4NAK3Sen8oRHxzzqPmxQRo+rcnhoRWy7mNf8GHFLn9g+Aq+o+YFHvOSL2AEYA313I6PF6EfFI6Xd4T0T0WNJ7Kd23VUQ8Vfq5G4AuiwoeERtExAOl39f7pcfXtUtEvBIRH0XEHyMiSj/XLiJ+FhFTIuK90me9cum+KyPipNL3a5dGoY+q83ofRlZvtLF0+erJEfFcKc8NEdGlzv3DIuKdiHg7In68qEt8I+I3wPbAH0q/0z8s6f2Ufu6wiHihdN/dEbHeYn5vi/v9vx4RwyNiYum5/hoRXSKiK/B/wFpRe7XAWlFnhD1qR+0PLf25+ygifhIRg0u/l4/rvp+I6B8R/46ID0qf3zURscqici/m/XSPiNERcUHd34kkqT4LqyS1Ht8ErgdWAW4D/gC56AD/BJ4F1gZ2Bo6PiN2b6XV/ABwGrAXMAy4ove6GwHXA8UBP4E7gnxHRCXgA2L5UwnoDHYGvln6uH9ANeG4xr3k1cEBEtC8Vl+7AEzV3Lu49p5TuAs4AbljI6PH3gEOBXkAn4OQlvZfS+7mVXKJXA24Evr2Y7L8C7gFWBdYBLmxw/17AYGALYChQ8zn9sHQMAWp+RzVF6gFgx9L3XwMml74C7AA8lFJKi8gzFNgDWB/4Quk1aor9icAuwAZ1nq+RlNJ/Aw8Bx5R+p8cs6f1ExLfI/3CwH/l3+hD5d9zIEv4s1Tio9Nz9gQ2Bn6WUZgJfB96uc7XA24t4G9sAA4DvAucD/11675sCQyOi5v0HcCb5z/smwLrAzxf1u1nE+1kduA94JKV07GI+G0lq8yysktR6PJxSujOlNJ9cnmqK2GCgZ0rplymlOSmlycClwAGLea6TSyNLNceVi3ns31JK40vl4HTyX+7bk//if0dK6d6U0lzgHGAF4CulDNOBLclF6G7grYjYuHT7oZTSgsW85pvAS+RCcQgNRleX8T0D/DWl9HJK6TNgVCkfi3svwLbkwn1+SmluSukmYMxiXmMusB6wVkppdkrp4Qb3n5VS+jil9AYwuk6Gg4DzUkqTU0ozgOHk0t6BOv8AQC6oIyn9AwD59/nAYvJckFJ6O6X0Ibnk17ze0NLvY0JKaRbwi8U8x+Is6v0cAZyZUnohpTSP/I8IWy5ilHVxv/8af0gpTS29j98ABy5lzl+VPo97gJnAdSml91JKb5HL9FYAKaVJpRyfp5SmAeexmDK/EGuRP48bU0o/W8qMktTmWFglqfX4T53vZwFdSmVmPfIlkf+/gJJHttZYzHOdk1Japc5xyGIeO7XO91PI5a0H+S/mU2ruKBXQqeQRT6gdFdyh9P395L/4L6lg1biKPBp4IHnEta5lec/Q+HfYrfT94t7LWsBbDUbJprBow8ijdE+WLnE9bFkylL7vAKyRUnoVmEEug9sDtwNvR8RGLPn3ubjXq/vZ1v1+aSzq+dcDfl/n8/mQ/HtZm8aW9GepYb4ppZ9ZGu/W+f6zhdzuBhARvSLi+oh4KyI+Jf/Z60HTfYNctv+8lPkkqU2ysEpS6zcVeK1BAe2eUtqzmZ5/3Trf9yGPIL4PvE0uJQCU5umtC7xVOlVTWLcvff8AS1dYbyb/5X9ySqlhQVzSe17aSzAX917eAdZuMA+xz6KeKKX0n5TSf6WU1iKPMv5pYfNCl5Sh9BrzqC1WDwDfATqVRgUfIF+uvSrwTBOev6F3yJcs11h3UQ8sWdrf6VTgiAaf0QoppUcX8tgl/VlqmK9P6WeWJdeSnFl6zi+klFYCvk8u2k11KXAXcGdpjq0kaTEsrJLU+j0JfBoRp0bECqV5n5tFxOBmev7vR8TAiFgR+CVwU+my5FHANyJi54joCJwEfA7UFJIHyPMxV0gpvUm+7HIPYHXg6SW9aOkS5J2AhW1PsqT3/C7Qt3QJbVMs7r08Ri6Ox0ZEh4jYD/jSop4oIvaPiJoi+BG5/MxvQobrgBMiYv2I6EbtPNx5pfsfAI4BHizdvh/4KflS8aY8f0OjgEMjYpPSZ/s/S3j8u+S5tU31Z2B4RGwKEBErR8T+i8myuD9LAEdHxDqRF/0aAdQsZvUusHqUFqhqBt3Jo9kfR8TawCnL8BzHkC9pvz0iVmimXJLUKllYJamVK5WVvcmXi75GHv38C7C4v8APi/r7sL6/mMf+DbiCfOlnF+DY0uu+RB59urD0mnuTt+OZU7r/ZfJf/B8q3f6UvFjQI00tWCmlsaXLYZf2Pd9Y+vpBNNjTdRGvs8j3Uno/+5EvT/6IPN/y74t5usHAExExg7w41nEppaZsyXM5+Xf9YOk9zSYX0hoPkMtUTWF9GFixzu2lklL6P/ICWqOBSeRiDrkoLszvge+UVtm9oAnPfwtwNnB96dLa8eQFkhb22MX+WSq5lryY1eTS8evSz75ILvuTS5cfL+2lwg39Avgi8AlwB4v/rBeqdPn44eRR5n9EnZWZJUn1hQvTSZKkJSmtxjwe6FxnVLciRMTrwI9TSv8qOoskqXk5wipJkhYqIvYtbd2zKnk09J+VVlYlSa2bhVWSJC3KEcA04FXyPNsji40jSWprvCRYkiRJklSRHGGVJEmSJFUkC6skSZIkqSJ1KDpAU/To0SP17du36BiSJEmSpDIYN27c+ymlng3PV0Vh7du3L2PHji06hiRJkiSpDCJiysLOe0mwJEmSJKkiWVglSZIkSRXJwipJkiRJqkgWVkmSJElSRbKwSpIkSZIqkoVVkiRJklSRLKySJEmSpIpkYZUkSZIkVSQLqyRJkiSpIllYJUmSJEkVycIqSZIkSapIFlZJkiRJUkWysEqSJEmSKpKFVZIkSZJUkSyskiRJkqSKZGFdDiNHwujR9c+NHp3PS5IkSZKWj4V1OQweDEOHwrXXQkq5rA4dms9LkiRJkpZPh6IDVLMhQ+C00+Cgg+COO+Cee2DUqHxekiRJkrR8HGFdTsccA92751HWI4+0rEqSJElSc7GwLqdHH4UFC/L3F17YeE6rJEmSJGnZWFiXQ82c1VGjYPXVYeON821LqyRJkiQtPwvrchgzJpfVPfeE446Dxx+HM8/M5yVJkiRJyydSSkVnWKJBgwalsWPHFh1jsT76CPr0gb32guuuKzqNJEmSJFWPiBiXUhrU8LwjrM1k1VXhqKPyiOukSUWnkSRJkqTqZ2FtRiecAB07wsiRRSeRJEmSpOpnYW1Ga64Jhx0GV1wBb71VdBpJkiRJqm4W1mZ2yil5m5tzzy06iSRJkiRVNwtrM1t/ffje9+Dii+H994tOI0mSJEnVy8JaBqedBrNmwQUXFJ1EkiRJkqqXhbUMBg6EffeFCy+E6dOLTiNJkiRJ1cnCWibDh8PHH8Of/1x0EkmSJEmqThbWMhk8GHbZJS++NHt20WkkSZIkqfpYWMtoxAh4913461+LTiJJkiRJ1cfCWkY77gjbbgsjR8LcuUWnkSRJkqTqYmEto4g8yvr663D99UWnkSRJkqTqUrbCGhHrRsToiHghIiZExHGl8z+PiLci4pnSsWe5MlSCb3wDNt8czjoLFiwoOo0kSZIkVY9yjrDOA05KKW0CbAscHREDS/f9LqW0Zem4s4wZCteuXV4xeOJEuO22otNIkiRJUvUoW2FNKb2TUnqq9P104AVg7XK9XiXbf3/o1w/OOANSKjqNJEmSJFWHFpnDGhF9ga2AJ0qnjomI5yLi8ohYtSUyFKlDBzj1VBgzBu67r+g0kiRJklQdyl5YI6IbcDNwfErpU+AioD+wJfAOcO4ifu7wiBgbEWOnTZtW7phld8gh0Lt3HmWVJEmSJC1ZWQtrRHQkl9VrUkp/B0gpvZtSmp9SWgBcCnxpYT+bUrokpTQopTSoZ8+e5YzZIjp3hpNPhtGj4fHHi04jSZIkSZWvnKsEB3AZ8EJK6bw653vXedi+wPhyZag0hx8Oq60GZ55ZdBJJkiRJqnzlHGH9KnAwsFODLWxGRsTzEfEcMAQ4oYwZKkq3bnDssXm14OefLzqNJEmSJFW2SFWwbO2gQYPS2LFji47RLD78EPr0gX32gWuuKTqNJEmSJBUvIsallAY1PN8iqwSr1mqrwZFHwvXXw6uvFp1GkiRJkiqXhbUAJ56Yt7oZObLoJJIkSZJUuSysBejdGw47DK64At5+u+g0kiRJklSZLKwFOeUUmD8fzjtvyY+VJEmSpLbIwlqQfv3ggAPgz3+GDz4oOo0kSZIkVR4La4FOOw1mzoQLLyw6iSRJkiRVHgtrgTbbLG9vc8EFMH160WkkSZIkqbJYWAs2fDh89BFccknRSSRJkiSpslhYC7bNNrDzznDuuTB7dtFpJEmSJKlyWFgrwIgR8M47cOWVRSeRJEmSpMphYa0AQ4bAl74EZ58N8+YVnUaSJEmSKoOFtQJE5FHW116DG24oOo0kSZIkVQYLa4XYe2/YdFM480xYsKDoNJIkSZJUPAtrhWjXLq8YPGEC3H570WkkSZIkqXgW1gry3e/C+uvDb34DKRWdRpIkSZKKZWGtIB06wLBh8OSTMHp00WkkSZIkqVgW1grzwx/CmmvCGWcUnUSSJEmSimVhrTBdusBJJ8F998ETTxSdRpIkSZKKY2GtQEccAauumlcMliRJkqS2ysJagbp3h2OPhX/8I68aLEmSJEltkYW1Qv30p9C1K5x1VtFJJEmSJKkYFtYKtfrq+dLg666DyZOLTiNJkiRJLc/CWsFOPBHat4ff/rboJJIkSZLU8iysFWzttfM2N5dfDu+8U3QaSZIkSWpZFtYKN2wYzJsHv/td0UkkSZIkqWVZWCtc//5wwAFw0UXw4YdFp5EkSZKklmNhrQKnnQYzZsAf/lB0EkmSJElqORbWKrD55rD33vD73+fiKkmSJEltgYW1SowYkS8JvuSSopNIkiRJUsuwsFaJbbeFIUPg3HPh88+LTiNJkiRJ5WdhrSIjRsDbb8NVVxWdRJIkSZLKz8JaRXbeGQYPhrPPzlvdSJIkSVJrZmGtIhEwfDi8+irceGPRaSRJkiSpvCysVWaffWCTTeCMM2DBgqLTSJIkSVL5WFirTLt2eZR1/Hi4446i00iSJElS+VhYq9ABB0DfvnmUNaWi00iSJElSeVhYq1DHjjBsGDz+ODzwQNFpJEmSJKk8LKxV6tBDYY018iirJEmSJLVGFtYq1aULnHgi3HsvjBlTdBpJkiRJan4W1ir2k5/AKqvAmWcWnUSSJEmSmp+FtYqttBL89Kdwyy0wcWLRaSRJkiSpeVlYq9yxx8KKK8LZZxedRJIkSZKal4W1yvXoAUccAddcA6+/XnQaSZIkSWo+FtZW4KSToF07+O1vi04iSZIkSc3HwtoKrL02HHIIXHYZ/Oc/RaeRJEmSpOZhYW0lhg2DuXPhd78rOokkSZIkNQ8LaysxYAAMHQp/+hN89FHRaSRJkiRp+VlYW5Hhw2HGDPjjH4tOIkmSJEnLz8LainzhC7DXXnD++TBzZtFpJEmSJGn5WFhbmeHD4YMP4NJLi04iSZIkScvHwtrKfOUr8LWvwTnnwOefF51GkiRJkpadhbUVGjEC3noL/va3opNIkiRJ0rKzsLZCu+4KW28NZ58N8+cXnUaSJEmSlo2FtRWKyKOskybBTTcVnUaSJEmSlo2FtZX61rdg443hjDMgpaLTSJIkSdLSs7C2Uu3awWmnwXPPwZ13Fp1GkiRJkpaehbUV+973oE8f+M1vHGWVJEmSVH0srK1Yx44wbBg89hg8+GDRaSRJkiRp6VhYW7nDDoNeveDMM4tOIkmSJElLx8Layq2wApx4Itx9N4wbV3QaSZIkSWo6C2sbcOSRsPLKjrJKkiRJqi4W1jZgpZXgmGPg73+HF14oOo0kSZIkNY2FtY047jjo0gXOPrvoJJIkSZLUNBbWNqJnTzj8cLjmGpgypeg0kiRJkrRkFtY25KSTIALOOafoJJIkSZK0ZBbWNmTddeEHP4C//AXefbfoNJIkSZK0eBbWNmbYMPj8czj//KKTSJIkSdLiWVjbmA03hP33hz/+ET7+uOg0kiRJkrRoFtY2aPhwmD49l1ZJkiRJqlQW1jZoyy1hzz3zZcGzZhWdRpIkSZIWzsLaRo0YAe+/nxdgkiRJkqRKZGFto776VdhhB/jtb2HOnKLTSJIkSVJjFtY2bPhwePNNuPrqopNIkiRJUmMW1jZs991hq63grLNg/vyi00iSJElSfRbWNiwiz2V95RW4+eai00iSJElSfRbWNm7ffWGjjeDMMyGlotNIkiRJUi0LaxvXvj2cdho88wzcdVfRaSRJkiSploVVHHQQ9OkDZ5xRdBJJkiRJqmVhFR07wsknw8MPw0MPFZ1GkiRJkjILqwD40Y+gZ09HWSVJkiRVjrIV1ohYNyJGR8QLETEhIo4rnV8tIu6NiFdKX1ctVwY13Yorwgkn5HmsTz9ddBpJkiRJKu8I6zzgpJTSJsC2wNERMRA4DbgvpTQAuK90WxXgqKNgpZXyisGSJEmSVLSyFdaU0jsppadK308HXgDWBvYBriw97ErgW+XKoKWz8spwzDFw003w0ktFp5EkSZLU1rXIHNaI6AtsBTwBrJFSegdyqQV6tUQGNc1xx0HnznD22UUnkSRJktTWlb2wRkQ34Gbg+JTSp0vxc4dHxNiIGDtt2rTyBVQ9vXrBf/0X/O1v8MYbRaeRJEmS1JaVtbBGREdyWb0mpfT30ul3I6J36f7ewHsL+9mU0iUppUEppUE9e/YsZ0w1cPLJ+es55xSbQ5IkSVLbVs5VggO4DHghpXRenbtuAw4pfX8I8I9yZdCy6dMHDj4Y/vIXeG+h/5wgSZIkSeVXzhHWrwIHAztFxDOlY0/gLGDXiHgF2LV0WxXm1FNh9mz4/e+LTiJJkiSprYqUUtEZlmjQoEFp7NixRcdoc4YOhbvvznNZV1656DSSJEmSWquIGJdSGtTwfIusEqzqNHw4fPop/OlPRSeRJEmS1BZZWLVIW20Fe+wBv/sdzJpVdBpJkiRJbY2FVYs1YgRMmwaXX150EkmSJEltjYVVi7X99rDddjByJMyZU3QaSZIkSW2JhVVLNGIETJ0K115bdBJJkiRJbYmFVUu0xx6w5ZZw1lkwf37RaSRJkiS1FRZWLVFEXjH4pZfglluKTiNJkiSprbCwqkm+/W0YMADOOAOqYOteSZIkSa2AhVVN0r49nHYaPP003HNP0WkkSZIktQUWVjXZ978P66yTR1klSZIkqdwsrGqyTp3glFPgwQfh4YeLTiNJkiSptbOwaqn8+MfQoweceWbRSSRJkiS1dhZWLZUVV4Tjj4c774Rnnik6jSRJkqTWzMKqpXb00dC9u6OskiRJksrLwqqltsoqubTeeCO8/HLRaSRJkiS1VhZWLZPjj4fOnWHkyKKTSJIkSWqtLKxaJmuskRdguuoqmDq16DSSJEmSWiMLq5bZySdDSnDuuUUnkSRJktQaWVi1zNZbDw46CC65BKZNKzqNJEmSpNbGwqrlcuqpMHs2XHBB0UkkSZIktTYWVi2XTTaB/faDCy+ETz8tOo0kSZKk1sTCquU2fDh88glcdFHRSSRJkiS1JhZWLbett4bdd4fzzoPPPis6jSRJkqTWwsKqZjF8OLz3Hlx+edFJJEmSJLUWFlY1ix12gK98BUaOhLlzi04jSZIkqTWwsKpZRMCIEfDGG3DddUWnkSRJktQaWFjVbPbcE77wBTjzTFiwoOg0kiRJkqqdhVXNpmaU9cUX4dZbi04jSZIkqdpZWNWsvvMd2GADOOMMSKnoNJIkSZKqmYVVzap9ezj1VBg3Du69t+g0kiRJkqqZhVXN7uCDYe2181xWSZIkSVpWFlY1u86d4eST4f774dFHi04jSZIkqVpZWFUW//VfsPrqjrJKkiRJWnYWVpVF165w/PFw++3w7LNFp5EkSZJUjSysKpujj4Zu3eCss4pOIkmSJKkaWVhVNquuCkcdBaNGwaRJRaeRJEmSVG0srCqrE06Ajh1h5Miik0iSJEmqNhZWldWaa8KPfgRXXAFvvVV0GkmSJEnVxMKqsjvlFFiwAM49t+gkkiRJkqqJhVVl17cvHHQQXHwxvP9+0WkkSZIkVQsLq1rEqafCrFlwwQVFJ5EkSZJULSysahEDB8K++8KFF8L06UWnkSRJklQNLKxqMcOHw8cfw5//XHQSSZIkSdXAwqoWM3gw7LprXnxp9uyi00iSJEmqdBZWtagRI+Ddd+Gvfy06iSRJkqRKZ2FVi/ra12DbbWHkSJg7t+g0kiRJkiqZhVUtKiKPsr7+Olx/fdFpJEmSJFUyC6ta3De+AZtvDmedBQsWFJ1GkiRJUqWysKrFtWuXVwyeOBFuu63oNJIkSZIqlYVVhdh/f+jfH844A1IqOo0kSZKkSmRhVSE6dIBTT4UxY+C++4pOI0mSJKkSWVhVmB/8ANZaK4+ySpIkSVJDFlYVpnNnOOkkGD0aHn+86DSSJEmSKo2FVYU6/HBYbTU488yik0iSJEmqNBZWFapbNzjuuLxa8PPPF51GkiRJUiWxsKpwxxyTi+tZZxWdRJIkSVIlsbCqcKutBkceCddfD6++WnQaSZIkSZXCwqqKcMIJ0LEjjBxZdBJJkiRJlcLCqorQuzcceihccQW8/XbRaSRJkiRVAgurKsYpp8D8+XDeeUUnkSRJklQJmlxYI6J9RKwVEX1qjnIGU9vTrx8ceCD8+c/wwQdFp5EkSZJUtCYV1oj4KfAucC9wR+m4vYy51EaddhrMnAkXXlh0EkmSJElFa+oI63HARimlTVNKm5eOL5QzmNqmTTeFffaBCy6A6dOLTiNJkiSpSE0trFOBT8oZRKoxfDh89BFcfHHRSSRJkiQVqUMTHzcZuD8i7gA+rzmZUnJ5HDW7bbaBnXfOiy8dcwx06VJ0IkmSJElFaOoI6xvk+audgO51DqksRoyAd96BK68sOokkSZKkokRKqekPjugOpJTSjPJFamzQoEFp7NixLfmSKlhK8OUvw3vvwcsvQ4emXgsgSZIkqepExLiU0qCG55u6SvBmEfE0MB6YEBHjImLT5g4p1YjIo6yvvQY33FB0GkmSJElFaOolwZcAJ6aU1ksprQecBFxavlgS7LVXXjX4zDNhwYKi00iSJElqaU0trF1TSqNrbqSU7ge6liWRVNKuXV4xeMIEuN1dfyVJkqQ2p6mFdXJEnB4RfUvHz4DXyhlMAvjud2H99eE3v8nzWiVJkiS1HU0trIcBPYG/A7eUvj+0XKGkGh06wKmnwpNPwujRS368JEmSpNZjqVYJLoqrBLdts2dDv34wcCD8619Fp5EkSZLU3JZpleCIOL/09Z8RcVvDo0xZpXq6dIGTToL77oMnnig6jSRJkqSWstgR1ojYOqU0LiK+trD7U0oPlC1ZHY6wavp0WG892GEHuPXWotNIkiRJak7LNMKaUhpX+nbLlNIDdQ9gyzLklBaqe3c49lj4xz/yqsGSJEmSWr+mLrp0yELO/bAZc0hL9NOfQteucNZZRSeRJEmS1BKWNIf1wIj4J9CvwfzV0cAHLRNRylZfHX7yE7juOpg8ueg0kiRJksqtwxLufxR4B+gBnFvn/HTguXKFkhblxBPhwgvht7+Fiy4qOo0kSZKkclrSHNYpwEPAzAZzWJ9KKc1rmYhSrbXWgh/+EC6/HN55p+g0kiRJksppiXNYU0rzgVkRsXIL5JGWaNgwmDcPzjuv6CSSJEmSyqmpiy7NBp6PiMsi4oKaY3E/EBGXR8R7ETG+zrmfR8RbEfFM6dhzecKrberfHw44AP78Z/jww6LTSJIkSSqXphbWO4DTgQeBcXWOxbkC2GMh53+XUtqydNzZ1KBSXaedBjNmwB/+UHQSSZIkSeWypEWXAEgpXRkRnYANS6deSinNXcLPPBgRfZczn7RQm28O3/wm/P73eSGmbt2KTiRJkiSpuTVphDUidgReAf4I/Al4OSJ2WMbXPCYinitdMrzqYl7z8IgYGxFjp02btowvpdZs+PB8SfAllxSdRJIkSVI5REppyQ+KGAd8L6X0Uun2hsB1KaWtl/BzfYHbU0qblW6vAbwPJOBXQO+U0mFLev1BgwalsWPHLjGn2p6ddoKXXsr7snbuXHQaSZIkScsiIsallAY1PN/UOawda8oqQErpZaDj0oZIKb2bUpqfUloAXAp8aWmfQ6prxAh4+2246qqik0iSJElqbk0trGNLKwTvWDouZcmLLjUSEb3r3NwXGL+ox0pNsfPOMHgwnH123upGkiRJUuvR1MJ6JDABOBY4DpgIHLG4H4iI64DHgI0i4s2I+BEwMiKej4jngCHACcucXAIi8ijrq6/CjTcWnUaSJElSc2rqHNbjUkq/X9K5cnEOqxZnwYK8anC7dvDss/mrJEmSpOqxvHNYD1nIuR8uVyKpmbRrl1cMHj8e7rij6DSSJEmSmstiC2tEHBgR/wTWj4jb6hz3Ax+0SEKpCQ44APr2hTPOgCZcNCBJkiSpCnRYwv2PAu8APYBz65yfDjxXrlDS0urQAYYNg6OOggcegB13LDqRJEmSpOW12BHWlNKUlNL9wC7AQymlB8gFdh0gyh9ParpDD4U11sijrJIkSZKqX1PnsD4IdImItYH7gEOBK8oVSloWXbrASSfBvffCmDFFp5EkSZK0vJpaWCOlNAvYD7gwpbQvMLB8saRlM3s2dO0KZ55Ze270aBg5srhMkiRJkpZNkwtrRHwZOAioWYd1SfNfpRa33XZ50aVbboGJE3NZHToUBg8uOpkkSZKkpdXUwno8MBy4JaU0ISL6AaPLlkpaRkOGwDXX5O933RX23RdGjcrnJUmSJFWXJhXWlNIDKaVvppTOLt2enFI6trzRpGXzrW/BPvvA22/DJ5/A6afD7bfDggVFJ5MkSZK0NJa0D+v5pa//bLAP620RcVuLJJSW0ujR8MgjcNpp0K0bTJoEe+8NW2wBV18Nc+cWnVCSJElSUyxphPVvpa/nkPdhbXhIFaVmzuqoUXnhpdtug/nzYcSIfP/BB8MGG8CFF8KsWcVmlSRJkrR4S9qHdVzp6wPARGBi6fLgB0rnpIoyZkz9OatDhuTbK68Mzz2XLw3u0weOPRbWWw9+9Sv48MNiM0uSJElauEgpLfrOiAD+FzgGCHLBnUfe2uaXLZIQGDRoUBo7dmxLvZzagIcfhrPPzgW2a1c44gg44QRYZ52ik0mSJEltT0SMSykNanh+SZcEHw98FRicUlo9pbQqsA3w1Yg4ofljSi1ju+3gn//Mo6777gu//z306wc/+hG8+GLR6SRJkiTBkgvrD4ADU0qv1ZxIKU0Gvl+6T6pqm28Of/tbXpjpiCPguutg4ED49rfhySeLTidJkiS1bUsqrB1TSu83PJlSmgZ0LE8kqeX17ZsXYpoyBX72M/j3v2GbbWDnneHee2ExV85LkiRJKpMlFdY5y3ifVJV69oRf/hLeeAPOPTdfHrzbbrD11nnxpvnzi04oSZIktR1LKqxbRMSnCzmmA5u3RECpCN27w4knwuTJcNllMHMmfPe7sPHGcMklMHt20QklSZKk1m9J29q0TymttJCje0rJS4LV6nXuDIcdBhMnws03w6qr5rmu668PI0fCp58WnVCSJElqvZY0wioJaN8e9tsPnngC7rsvL9Z06ql5T9cRI+Ddd4tOKEmSJLU+FlZpKUTATjvBPffA2LGw++5w1lmw3npw1FH5EmJJkiRJzcPCKi2jrbeGG26Al16CQw7Jc10HDIDvfQ+efbbodJIkSVL1s7BKy2nAALj4Ynj9dTj5ZLj9dthyS/j61+GBB9wSR5IkSVpWFlapmfTuDWefnbfEOeMMeOop2HFH+MpX4B//gAULik4oSZIkVRcLq9TMVlkFhg/PI65/+lNekOlb34LNNoMrr4Q57mAsSZIkNYmFVSqTFVaAI4+El1+Ga6+Fjh3hhz+EDTaA88+HGTOKTihJkiRVNgurVGYdOsCBB8Izz8Cdd0K/fnDCCXll4Z//HD74oOiEkiRJUmWysEotJCIvxHT//fDoo7D99vCLX+S9XI8/Ps99lSRJklTLwioV4MtfhltvhQkTYP/94Y9/hP798yXDEycWnU6SJEmqDBZWqUADB8IVV8Crr8LRR8ONN8Kmm+ZFmh5/vOh0kiRJUrEsrFIF6NMnL8Q0ZQr87//CQw/lUdgdd4S77nIvV0mSJLVNFlapgvTokRdimjIFfve7PPL69a/DVlvBddfBvHlFJ5QkSZJajoVVqkDduuWFmF59Ff7617x36/e+BxtuCBddBJ99VnRCSZIkqfwsrFIF69QpL8Q0fnxepKlXLzjqKOjbF848Ez7+uNh8kiRJUjlZWKUq0K4d7LMPPPZY3hbni1+EESPy3NdTT4V33ik6oSRJktT8LKxSFYmAr30N/u//4Omn4RvfgHPOySOuRxwBkyYVnVCSJElqPhZWqUptuWVeiOnll+Gww+DKK2GjjeC734Wnnio6nSRJkrT8LKxSlevfPy/E9PrrMGxY3gZn661h993h3/92SxxJkiRVLwur1EqsuWZeiOmNN+Dss+G552DnnWGbbeDvf4cFC4pOKEmSJC0dC6vUyqy8ch5pfe01uPhi+PBD+Pa3YeBAuPzyvEWOJEmSVA0srFIr1aULHH44vPQS3HADrLgi/OhH0K8fnHceTJ9edEJJkiRp8SysUivXvj0MHQrjxsHdd8OGG8JJJ8F668H//A9Mm1Z0QkmSJGnhLKxSGxEBu+2WF2J6/HHYcUf49a9zcf3pT/OiTZIkSVIlsbBKbVDNQkwTJ8IBB+S5rhtsAAcfDOPHF51OkiRJyiysUhu28cZ5IabJk+G44+CWW2DzzWHvveGRR4pOJ0mSpLbOwiqJddaBc8/NW+L88pfw2GOw3Xaw/fZwxx3u5SpJkqRiWFgl/X+rrQannw5TpsAFF+QCu9desMUWcM01MG9e0QklSZLUllhYJTXStWteiGnSJLjqKliwAL7/fRgwAP74R5g1q+iEkiRJagssrJIWqWPHvBDTc8/BbbfBWmvBMcdA3755heGPPio6oSRJklozC6ukJWrXrnYhpocegi99KV863KcPnHwyvPVW0QklSZLUGllYJS2V7baD22+HZ5+FffaB88+H9deHH/8YXnqp6HSSJElqTSyskpbJF74AV18Nr7wChx+eF2XaZBP4zndgzJii00mSJKk1sLBKWi7rrw9/+ENeWXjECLjvvnzJ8C67wL/+BWefDaNH1/+Z0aNh5Mhi8kqSJKl6WFglNYtevfJCTFOmwG9/CxMnwq67wmWX5UuH//Wv/LjRo2HoUBg8uNi8kiRJqnwWVknNaqWV8kJMr70Gl16az02fDrvvnvd0HToURo2CIUOKzSlJkqTKZ2GVVBadO+eFmF54AW66CdZYA+64A2bMyF9ffbXohJIkSap0FlZJZdW+Pay2GsydCz/4ASxYAL/7HQwYAHvumcvrggVFp5QkSVIlsrBKKquaOaujRsGVV8Jdd8Eqq8DBB8PTT+fLhAcMgHPOgQ8/LDqtJEmSKomFVVJZjRlTf87qkCH5EuFNN80LNF1/Pay9NpxySv562GEwblyxmSVJklQZIqVUdIYlGjRoUBo7dmzRMSSV0fPPwx//mPd2nTkTttkGjj4a9t8funQpOp0kSZLKKSLGpZQGNTzvCKukirD55vDnP8Nbb8Hvfw8ff5znvK67LgwfnkdjJUmS1LZYWCVVlJVXhmOPzasL33svbLcdjBwJ/frl/VzvucdFmiRJktoKC6ukihQBu+wCt9yS93Q99VR47LG8n+smm9SOwkqSJKn1srBKqnh9+sAZZ8DUqfC3v+Vtco4/Pi/SdMQR8NxzRSeUJElSOVhYJVWNzp3h+9/PI63jxsEBB8BVV8EWW8D22+cVh+fMKTqlJEmSmouFVVJV+uIX4bLL8iJN55wDb78NBx4I660H//u/+bwkSZKqm4VVUlVbbTU46SR45RW4445cZH/1q1xc998f7r8fqmD3LkmSJC2EhVVSq9CuHey5Zy6tkybBCSfAv/8NQ4bkLXP+9CeYPr3olJIkSVoaFlZJrU6/fvDb38Kbb8Lll0OXLnD00XmRpmOOyVvmSJIkqfJZWCW1WiusAIceCmPGwOOPw7e+BZdeCgMHwk47wc03w7x5RaeUJEnSolhYJbV6EbDNNnlF4alT8xY5r74K3/kOrL8+/PrX8O67RaeUJElSQxZWSW1Kr14wfDhMngy33gqbbAKnnw7rrgvf+x488oiLNEmSJFUKC6ukNql9e9hnH7jnHnjxRTjqKLjzTthuO9hqq3zp8MyZRaeUJElq2yysktq8jTaC88/Pe7defDEsWACHHw7rrAMnnpi3zJEkSVLLs7BKUknXrrmoPvssPPgg7L47XHghbLgh7LEH/POfMH9+0SklSZLaDgurJDUQAdtvD9dfD2+8Ab/4BTz/PHzzm7DBBnD22fD++0WnlCRJav0srJK0GL17w//8D7z+Otx4I/TtC6edli8XPuQQePLJohNKkiS1XhZWSWqCjh3zNjijR8P48fCjH8Hf/563yxk8GK68EmbPLjqlJElS62JhlaSltOmm8Mc/5kWaLrwQZsyAH/4wj7qeeiq89lrRCSVJkloHC6skLaOVVoJjjoGJE+G+++BrX4Nzz4X+/WHvveGuu/KKw5IkSVo2ZSusEXF5RLwXEePrnFstIu6NiFdKX1ct1+tLUkuJgJ12gptvznNd//u/89zWr389b5lz3nnw0UdFp5QkSao+5RxhvQLYo8G504D7UkoDgPtKtyWp1VhnHfjVr2DqVLj2WujVC046CdZeG378Y3j66aITSpIkVY+yFdaU0oPAhw1O7wNcWfr+SuBb5Xp9SSpSp05w4IHwyCO5pB50UC6wX/wifPWr+fs5c4pOKUmSVNlaeg7rGimldwBKX3u18OtLUovbcku49NK8SNN558F77+UCu+668LOf5dFYSZIkNVaxiy5FxOERMTYixk6bNq3oOJK03FZdFU44AV56KS/ItM02cMYZsP76sN9+eeGmlIpOKUmSVDlaurC+GxG9AUpf31vUA1NKl6SUBqWUBvXs2bPFAkpSubVrB7vvDrfdBpMnw8knw4MPwi67wMCB8Ic/wKefFp1SkiSpeC1dWG8DDil9fwjwjxZ+fUmqKH37wllnwZtvwhVXQPfu8NOf5kWajjoKJkwoOqEkSVJxyrmtzXXAY8BGEfFmRPwIOAvYNSJeAXYt3ZakNq9LFzjkkLwdzpNPwre/DZdfDpttBjvuCDfeCHPnFp1SkiSpZUWqgglTgwYNSmPHji06hiS1qPffz6X1oovy/q5rrQWHH56P3r2LTidJktR8ImJcSmlQw/MVu+iSJLV1PXrAsGEwaVKe77r55vDzn0OfPvDd7+Z5r1Xwb46SJEnLzMIqSRWufXvYe++8svDLL+c5rvfcA1/7GmyxBVx8McyYUXRKSZKk5mdhlaQqMmBA3sv1rbfy3q7t28NPfpIXaTruuLxljiRJUmthYZWkKrTiivDjH8NTT8Ejj8Bee+W5rhtvDLvuCrfeCvPmFZ1SkiRp+VhYJamKRcBXvgLXXANTp8Kvfw0vvgj77gv9+8OZZ8K0aUWnlCRJWjYWVklqJdZYA/77v+G11+Dmm2GDDWDECFhnHTj4YHj8cRdpkiRJ1cXCKkmtTIcOsN9+cN99MHFi3gbnH/+AL38ZBg3KW+X85jcwenT9nxs9GkaOLCazJEnSwlhYJakV22QTuPDCvEjTn/4En38OP/oRnHUWfOMbcPXV+XGjR8PQoTB4cLF5JUmS6rKwSlIb0L07HHkkPP98Lqd77JHL68EH531d99orz3/92teKTipJklTLwipJbUgE7Lgj3HgjvPEG7LBDXqxp1qy8PU7v3vCDH8C117pYkyRJKp6FVZLaqJdfznNcTz8dVlsNTjsNdtkF7rwTDjooL+L0pS/B//wPPPqo2+RIkqSWZ2GVpDaoZs7qqFHwy1/CTTfBX/6S93Z991148kn4xS+gY8e8QNNXvwo9e+afufzyPCdWkiSp3CJVwR4HgwYNSmPHji06hiS1GiNH5gWWhgypPTd6NIwZA8OG1X/sRx/lFYfvuisfNWV1881h993zfNjttoPOnVsuvyRJal0iYlxKaVCj8xZWSVJTpQQTJtSW1wcfhLlzYcUVYaedcnndYw/o37/opJIkqZpYWCVJzW7GDLj//lxe/+//YPLkfH6DDWrL6447QteuRaaUJEmVzsIqSSq7SZNqR19Hj86rD3fqlFcjrimwAwfm1YolSZJqWFglSS1q9mx4+OHaAjthQj6/zjq15XXnnWGVVQqNKUmSKoCFVZJUqKlT4e67c3m991749FNo3x6+/OXaArvVVtDO9eslSWpzLKySpIoxdy488UTt6Ou4cfl8z56w2265vO62G/TqVWxOSZLUMiyskqSK9d57cM89ubzefTe8/34+v/XWtaOv224LHToUm1OSJJWHhVWSVBUWLICnnqq9fPixx2D+fFh5Zdhll1xed98d1l236KSSJKm5WFglSVXp44/hvvtqLx9+8818ftNNa0dft98eOncuNKYkSVoOFlZJUtVLCSZOrC2vDz4Ic+bAiivCkCG1BXaDDYpOKkmSloaFVZLU6sycCfffX1tgJ03K5/v3ry2vO+4I3boVmVKSJC2JhVWS1OpNmlQ79/Xf/4ZZs6BTJ9huu9oCu9lmEFF0UkmSVJeFVZLUpnz+OTz8cO3o6/jx+fxaa9WW1112gVVXLTanJEmysEqS2rg336wdfb33XvjkE2jXLm+XU1Ngt946n5MkSS3LwipJUsm8efDEE7UFduzYvKBTjx6w2265vO62G6yxRtFJJUlqGyyskiQtwrRpedT1rrtyiX3vvXz+i1+sHX3ddlvo2LHYnJIktVYWVkmSmmDBAnjmmdq5r48+CvPnw0or5Tmve+wBu+8OffoUnVSSpNbDwipJ0jL45BO4777aAjt1aj6/ySa1o6877ABduhSbU5KkamZhlSRpOaUEL7xQW14feADmzIEVVsj7vdYU2AED3DpHkqSlYWGVJKmZzZyZS2tNgX3llXx+/fVry+uQIdC9e7E5JUmqdBZWSZLK7NVXa1ce/ve/c6Ht2BG22662wG6+uaOvkiQ1ZGGVJKkFff45PPJI7crDzz2Xz/fuXVted9kFVlstnx85EgYPziOyNUaPhjFjYNiwls8vSVJLWlRhdXt0SZLKoHNn2GmnXESffRbefBMuvxy23x5uuQW++13o2RO+8hX45S+hWzcYOjSXVMhfhw7NJVaSpLbKEVZJklrYvHl55LRm7uuYMXlBp+7d8yJOu+0GDz0E112XR2IlSWrtvCRYkqQK9f77cO+9ubzedBPMmpXPt2sH/fvnea+bbVZ7DBgAHToUm1mSpOa0qMLqf+4kSSpYjx5w4IGw5ppw551wyCFw9dWw774wYwaMHw+33goLFuTHd+qU94GtKbA1hbZPHxd0kiS1LhZWSZIqQM2c1VGj8sJL++9f//Znn8GLL8Lzz+cCO348PPggXHNN7XN07w6bbtp4RLZXr+LelyRJy8NLgiVJqgDLukrwxx/DhAm1Jfb55/Px4Ye1j+nVq36B3XzzXGzdH1aSVCmcwypJUhuRErz7bm2BrSmzEybkvWFrrLde/UuKN9sMNt44r3AsSVJLcg6rJEltRESeD7vmmnmv1xoLFsDrr9cW2JpCe/fdeeVigPbtYcMNG4/I9uuX75MkqSVZWCVJaiPatcvFs18/+OY3a8/PmQOvvFJ/RPapp/KKxTUXYnXpAgMHNh6RXXttF3qSJJWPlwRLkqSFmjkTJk5sPCL7zju1j1lllfqjsTXH6qsXFluSVIW8JFiSJC2Vrl3zQlCDB9c//8EHtQs91YzIXn99XgCqRu/ejS8rHjgwP6ckSU1lYZUkSUtl9dVhhx3yUSMlePvt+os8jR8PF10Es2fnx0TA+us3vqx4ww3z3rKSJDVkYZUkScstIs9nXXtt2GOP2vPz58PkyfUvKR4/Hu64I98H0LEjbLRR4xHZvn3zvFtJUtvlHFZJktTiPv8cXnqp8dY7r79e+5gVV8z7xTYckV1zTRd6kqTWxjmskiSpYnTuDF/4Qj7qmj69dn5szXHHHfDXv9Y+ZvXVF77Q0yqrtOhbkCS1AAurJEmqGN27w7bb5qOu995rvNDTVVflgltjnXXqX1K82WawySawwgot+x4kSc3HwipJkiper175GDKk9lxKMHVq44We/v3vvLcs5Dmw/fvXv6R4s81gwADo4N+CJKni+X/VkiSpKkVAnz75+MY3as/PmweTJjVe6OnWW2HBgvyYTp3y6GvDEdk+ferPjx05Mm/rU7cojx4NY8bAsGEt8jYlqU2zsEqSpFalQwfYeON8fOc7tec/+wxefLH+iOyDD8I119Q+pnv3vNBTTYFt3x723x9uvDGX1tGjYehQGDWq5d+XJLVFFlZJktQmrLACbLVVPur6+OPGCz39/e9w6aW1j9l55zz6+u678JOf5FL84Yew2mot+hYkqc1xWxtJkqQGUsrltKbAXnklPPNM3jN27tzax625Zu3WO5tuWnusvHJh0SWpKrmtjSRJUhNF5DK65pr5suA334TTT4eLLoILLsiFdMKE2uPSS2HWrNqfX3vt+gV2001h4EBYaaXi3pMkVSMLqyRJ0iLUnbM6ZEg+am6fckrt4xYsgClTai8trimyF10Es2fXPq5Pn4UX2a5dW/69SVI1sLBKkiQtwpgxtWUV8tdRo/L5uisHt2sH66+fj732qj0/fz689lr90dgJE/LWO59/Xvu4vn0bX1bsHrKS5BxWSZKkFjdvHrz6auMi+9JLtXNkI6Bfv/oldrPNYKONoEuXYvNLUnNzDqskSVKF6NAhF8+NNoL99qs9P3du3kO2psDWXF5855255EIezd1gg8aXFm+0Ud5fVpJaE0dYJUmSKtycOfDyy41HZF95Jc+fhbw41IYbNi6yAwbk1Y0lqZI5wipJklSlOnXKlwNvtln987Nn58uI65bYZ56Bm2/OW/NALqt1i2zNXNn+/fNIryRVMv9vSpIkqUp16QJbbJGPuj77DF58sX6RrVlAqkanTrDxxo1HZPv1y6O1klQJLKySJEmtzAorwFZb5aOumTPhhRfqF9lHH4Xrrqt9TJcueYXihkW2b988f1aSWpKFVZIkqY3o2hUGDcpHXdOnw8SJ9Yvs/ffD1VfXPmbFFWuLbN0tePr0ySsaS1I5uOiSJEmSFuqTTxoX2fHj4Z13ah/TrRsMHNh4RHaddSyykppuUYsuWVglSZK0VD76qPGKxRMmwLvv1j5mpZUal9hNN4XevS2ykhqzsEqSJKmsPvig/v6xNcf779c+ZpVVGq9YvOmm0KuXRVZqyyyskiRJKsR77zUejR0/Po/U1lh99YWPyPbsWf+5Ro6EwYNhyJDac6NH51WQhw1rmfcjqfm5D6skSZIK0atXPuqWzJTgP/9pXGSvvTbPna37s3ULbOfOsP/+cOON+flGj4ahQ+tv2SOp9bCwSpIkqcVF5PmsvXvDLrvUnk8J3n67/kjshAlw5ZV5NeMaO+8M662XS+8Pf5jve+GFvI9s584t/nYklYmXBEuSJKnipQRTp9YW2auvhmefzfvGzp5d+7h27fJWOwMG1D822ADWXx86dSruPUhaNC8JliRJUtWKyEW0T59cUs8+G04/HS66CK6/HtZcE155JR+TJuWv114LH39c+xzt2+dR2YZldsAA6NsXOvg3Y6ni+D9LSZIkVY26c1aHDMlHze3vf7/+Y1PKKxfXFNm6x6OP1r/EuEOHPAK7wQaNy+x66+WyK6nlWVglSZJUNcaMqS2rkL+OGpXP113UCfKobI8e+fjyl+vflxJMm7bwMvvggzBzZu1jO3bMc2NrLi2uW2bXXdcyK5WTc1glSZKkOmpWMK57eXHdy41nzap9bKdO0L9/4/myAwbAOuvkObWSlsw5rJIkSVIT1F3BeIcd6t9Xs4pxwxL7yitwzz31F4Dq0qVxma051lorv46kxbOwSpIkSU0UAWuvnY8dd6x/34IF8NZbjS8xfukluPNOmDOn9rErrLDw+bIDBuQFpCyzUmZhlSRJkppBu3Z5Tuu668JOO9W/b/58ePPNxmV2wgT45z9h7tzax3btuugy26uXZVZtSyGFNSJeB6YD84F5C7tWWZIkSWotarbUWW892GWX+vfNmwdvvNF4vuyzz8Ktt+b7a3TvvvD5sgMG5MWlLLNqbYocYR2SUnq/wNeXJEmSCtehQ16FuF8/2G23+vfNnQtTpjSeLzt2LNx0Ux65rbHyyo1HZGsK7eqrt+x7kpqLlwRLkiRJFapjx1w6N9gAvv71+vfNmQOvv974MuPHHoMbbshzamusuurCLzEeMABWWaUl35G0dIoqrAm4JyIScHFK6ZKCckiSJElVqVMn2HDDfDT0+efw2muNy+xDD8G11+bVjmv06LHoObMrrdRy70damEL2YY2ItVJKb0dEL+Be4KcppQcbPOZw4HCAPn36bD1lypQWzylJkiS1NrNnw+TJjcvsK6/khaHq6tWr8VzZmtvdu9c+buRIGDwYhgypPTd6NIwZA8OGtcz7UnVb1D6shRTWegEifg7MSCmds6jHDBo0KI0dO7blQkmSJElt0KxZ8OqrjefMvvJK3n+2rjXXrC2wETBqFJx3Hhx4IDz5JAwdms/VLbHSolRMYY2IrkC7lNL00vf3Ar9MKd21qJ+xsEqSJEnFmjmz8UrGNaX2P/+p/9h27WCzzWDQoNoR2ZqjW7di8quyLaqwFjGHdQ3glshrbncArl1cWZUkSZJUvK5dYYst8tHQ9Om5uP7mN3DzzbD55vmS4TvugHffrf/YmpHZmgJb9/u6lxlLUEBhTSlNBhbyx1ySJElSNereHT7+GB54AE4/HS66qPZy4OnTay8znjSpdpT2rrvgnXfqP88aa9QvsXXLrAtAtU1uayNJkiRpuYweXX/O6pAh9W9vuWU+GpoxI5fZmhJb8/Wee+CKK+o/tlevxiW25vuVV26BN6lCWFglSZIkLZcxY+ovsDRkSL49ZsziF13q1m3RlxnPnNm4zE6aBP/6F1x5Zf3H9uix8FFZ95mtfoWvEtwULrokSZIkqcasWbVb8zQstFOn1n/s6qsvfFR2wABYddVi8quxSlp0SZIkSZKW2Yor5lWIN9us8X2ffVa/zNYU2gcfhGuugbrjdauttujLjFdfveXejxbNwipJkiSp1VhhBdh003w0NHt2LrMNR2UffhiuvbZ+mV111YWPytaU2bzpicrNwipJkiSpTejSBQYOzEdDn39eW2brFtrHHoMbboAFC2ofu/LKi54z26OHZbY5WVglSZIktXmdO8Mmm+Sjoc8/h9dfbzxn9skn8+JSdcvsSistfFR2wADo2dMyu7QsrJIkSZK0GJ07w0Yb5aOhOXNymW24Nc/YsXDTTTB/fu1ju3df9D6za6xhmV0YC6skSZIkLaNOnWDDDfPR0Ny5jcvspEnw1FNw8831y2y3boueM7vmmm23zFpYJUmSJKkMOnbMpXPAAPj61+vfN3cuTJnSeM7sc8/BrbfCvHm1j+3atXGZrfm+d++Fl9mRI2Hw4Pr74I4enffGHTasLG+3LCyskiRJktTCOnasLZ4NzZsHb7zReGue8ePhttty2a2x4orQv3/jUdm+fWHo0DzHdsiQXFZrbleTSHXXbq5QgwYNSmPHji06hiRJkiQVat48mDq18ZzZSZPyKsdz5tQ+tlOnfNnxhhvCtGm15bUSRcS4lNKghucdYZUkSZKkKtGhA6y/fj523bX+ffPn15bZmiJ7++3wwgtw+umVW1YXp13RASRJkiRJy699+3wp8C67wE9+AnvtBR9+mMvqRRfly4KrjYVVkiRJklqZunNWf/nL/HXo0OorrRZWSZIkSWplxoypP2d1yJB8e8yYYnMtLRddkiRJkiQValGLLjnCKkmSJEmqSBZWSZIkSVJFsrBKkiRJkiqShVWSJEmSVJEsrJIkSZKkimRhlSRJkiRVJAurJEmSJKkiWVglSZIkSRXJwipJkiRJqkgWVkmSJElSRbKwSpIkSZIqkoVVkiRJklSRLKySJEmSpIpkYZUkSZIkVSQLqyRJkiSpIkVKqegMSxQR04ApRedYgh7A+0WHUD1+JpXHz6Qy+blUHj+TyuTnUnn8TCqTn0vlqYbPZL2UUs+GJ6uisFaDiBibUhpUdA7V8jOpPH4mlcnPpfL4mVQmP5fK42dSmfxcKk81fyZeEixJkiRJqkgWVkmSJElSRbKwNp9Lig6gRvxMKo+fSWXyc6k8fiaVyc+l8viZVCY/l8pTtZ+Jc1glSZIkSRXJEVZJkiRJUkWysC6HiLg8It6LiPFFZ1GtiFg3IkZHxAsRMSEijis6U1sXEV0i4smIeLb0mfyi6EzKIqJ9RDwdEbcXnUVZRLweEc9HxDMRMbboPIKIWCUiboqIF0v/bfly0ZnauojYqPS/kZrj04g4vuhcbV1EnFD67/z4iLguIroUnUkQEceVPpMJ1fi/Ey8JXg4RsQMwA7gqpbRZ0XmURURvoHdK6amI6A6MA76VUppYcLQ2KyIC6JpSmhERHYGHgeNSSo8XHK3Ni4gTgUHASimlvYrOo1xYgUEppUrfL6/NiIgrgYdSSn+JiE7AiimljwuOpZKIaA+8BWyTUppSdJ62KiLWJv/3fWBK6bOIGAXcmVK6othkbVtEbAZcD3wJmAPcBRyZUnql0GBLwRHW5ZBSehD4sOgcqi+l9E5K6anS99OBF4C1i03VtqVsRulmx9Lhv5YVLCLWAb4B/KXoLFKlioiVgB2AywBSSnMsqxVnZ+BVy2pF6ACsEBEdgBWBtwvOI9gEeDylNCulNA94ANi34ExLxcKqVi0i+gJbAU8UHKXNK116+gzwHnBvSsnPpHjnA8OABQXnUH0JuCcixkXE4UWHEf2AacBfS5fP/yUiuhYdSvUcAFxXdIi2LqX0FnAO8AbwDvBJSumeYlMJGA/sEBGrR8SKwJ7AugVnWioWVrVaEdENuBk4PqX0adF52rqU0vyU0pbAOsCXSpeoqCARsRfwXkppXNFZ1MhXU0pfBL4OHF2afqLidAC+CFyUUtoKmAmcVmwk1Shdov1N4Mais7R1EbEqsA+wPrAW0DUivl9sKqWUXgDOBu4lXw78LDCv0FBLycKqVqk0T/Jm4JqU0t+LzqNapUvp7gf2KDZJm/dV4Jul+ZLXAztFxNXFRhJASunt0tf3gFvI845UnDeBN+tcFXITucCqMnwdeCql9G7RQcQuwGsppWkppbnA34GvFJxJQErpspTSF1NKO5CnM1bN/FWwsKoVKi3wcxnwQkrpvKLzCCKiZ0SsUvp+BfJ/1F4sNFQbl1IanlJaJ6XUl3w53b9TSv5LeMEiomtpsThKl53uRr6cSwVJKf0HmBoRG5VO7Qy4iF/lOBAvB64UbwDbRsSKpb+L7UxeR0QFi4hepa99gP2osv/NdCg6QDWLiOuAHYEeEfEm8L8ppcuKTSXyyNHBwPOlOZMAI1JKdxYXqc3rDVxZWsmxHTAqpeQ2KlJjawC35L/r0QG4NqV0V7GRBPwUuKZ0+elk4NCC8wgozcfbFTii6CyClNITEXET8BT5ktOngUuKTaWSmyNidWAucHRK6aOiAy0Nt7WRJEmSJFUkLwmWJEmSJFUkC6skSZIkqSJZWCVJkiRJFcnCKkmSJEmqSBZWSZIkSVJFsrBKklQhIqJvRLjvqiRJJRZWSZIkSVJFsrBKklSBIqJfRDwdEYOLziJJUlEsrJIkVZiI2Ai4GTg0pTSm6DySJBWlQ9EBJElSPT2BfwDfTilNKDqMJElFcoRVkqTK8gkwFfhq0UEkSSqaI6ySJFWWOcC3gLsjYkZK6dqC80iSVBgLqyRJFSalNDMi9gLujYiZKaV/FJ1JkqQiREqp6AySJEmSJDXiHFZJkiRJUkWysEqSJEmSKpKFVZIkSZJUkSyskiRJkqSKZGGVJEmSJFUkC6skSZIkqSJZWCVJkiRJFcnCKkmSJEmqSP8PPBm65Y8Pb0UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "plt.plot(K, distortions, 'bx-')\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('Distortion')\n",
    "plt.title('The Elbow Method showing the optimal k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "29b2a5d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeanModel, clusters = k_means(feature_matrix=train_feature_matrix,\n",
    "                           num_clusters=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "dd5cc970",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km_obj.predict(test_feature_matrix[0])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "c7a64b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "key_df = pd.DataFrame(row_id_keys.items())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "e1a39d06",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_key_df=key_df[40:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "86ffc0bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>40</td>\n",
       "      <td>jiTmYb7pPM.7937506604855339.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>41</td>\n",
       "      <td>juE9npQetZ.5163804793372522.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>kr2ybHdzaR.9232864911517599.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>43</td>\n",
       "      <td>lDgbs8m1tf.08676825930990162.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>lR1GLF0dl3.9833018883394823.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>45</td>\n",
       "      <td>lZdgCU6d1U.12307985178981151.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>46</td>\n",
       "      <td>lsKjqDEeir.4889217778570776.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>47</td>\n",
       "      <td>m0xVYqvN8d.7758863033939478.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>48</td>\n",
       "      <td>npow2gShDJ.7019849575646654.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>49</td>\n",
       "      <td>p8IbJYSXxE.22562450196421968.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>50</td>\n",
       "      <td>rxt3ocjppG.7973298407205462.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>51</td>\n",
       "      <td>sGN3CBtN8o.35422772005643977.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>52</td>\n",
       "      <td>sHMzQdloFS.2681163681335329.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>53</td>\n",
       "      <td>tiMS4ch5zy.0033628833141092773.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54</td>\n",
       "      <td>vNPTbjqdSJ.7197300852931762.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>55</td>\n",
       "      <td>vPSU9hwW5E.2173308100445802.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>wZyhwhK1Oa.6182611089275415.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>57</td>\n",
       "      <td>ySljWqQVmz.9326708723080476.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>58</td>\n",
       "      <td>ygDG6ZzZ8T.25798542132023394.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>59</td>\n",
       "      <td>z6qfxfW6F4.4237818575016189.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>60</td>\n",
       "      <td>zu5R84I8ST.7841470681049597.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    key                            filename\n",
       "40   40     jiTmYb7pPM.7937506604855339.txt\n",
       "41   41     juE9npQetZ.5163804793372522.txt\n",
       "42   42     kr2ybHdzaR.9232864911517599.txt\n",
       "43   43    lDgbs8m1tf.08676825930990162.txt\n",
       "44   44     lR1GLF0dl3.9833018883394823.txt\n",
       "45   45    lZdgCU6d1U.12307985178981151.txt\n",
       "46   46     lsKjqDEeir.4889217778570776.txt\n",
       "47   47     m0xVYqvN8d.7758863033939478.txt\n",
       "48   48     npow2gShDJ.7019849575646654.txt\n",
       "49   49    p8IbJYSXxE.22562450196421968.txt\n",
       "50   50     rxt3ocjppG.7973298407205462.txt\n",
       "51   51    sGN3CBtN8o.35422772005643977.txt\n",
       "52   52     sHMzQdloFS.2681163681335329.txt\n",
       "53   53  tiMS4ch5zy.0033628833141092773.txt\n",
       "54   54     vNPTbjqdSJ.7197300852931762.txt\n",
       "55   55     vPSU9hwW5E.2173308100445802.txt\n",
       "56   56     wZyhwhK1Oa.6182611089275415.txt\n",
       "57   57     ySljWqQVmz.9326708723080476.txt\n",
       "58   58    ygDG6ZzZ8T.25798542132023394.txt\n",
       "59   59     z6qfxfW6F4.4237818575016189.txt\n",
       "60   60     zu5R84I8ST.7841470681049597.txt"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_key_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "655ba6dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>40</td>\n",
       "      <td>jiTmYb7pPM.7937506604855339.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>41</td>\n",
       "      <td>juE9npQetZ.5163804793372522.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>kr2ybHdzaR.9232864911517599.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>43</td>\n",
       "      <td>lDgbs8m1tf.08676825930990162.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>lR1GLF0dl3.9833018883394823.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>45</td>\n",
       "      <td>lZdgCU6d1U.12307985178981151.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>46</td>\n",
       "      <td>lsKjqDEeir.4889217778570776.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>47</td>\n",
       "      <td>m0xVYqvN8d.7758863033939478.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>48</td>\n",
       "      <td>npow2gShDJ.7019849575646654.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>49</td>\n",
       "      <td>p8IbJYSXxE.22562450196421968.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>50</td>\n",
       "      <td>rxt3ocjppG.7973298407205462.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>51</td>\n",
       "      <td>sGN3CBtN8o.35422772005643977.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>52</td>\n",
       "      <td>sHMzQdloFS.2681163681335329.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>53</td>\n",
       "      <td>tiMS4ch5zy.0033628833141092773.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54</td>\n",
       "      <td>vNPTbjqdSJ.7197300852931762.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>55</td>\n",
       "      <td>vPSU9hwW5E.2173308100445802.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>wZyhwhK1Oa.6182611089275415.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>57</td>\n",
       "      <td>ySljWqQVmz.9326708723080476.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>58</td>\n",
       "      <td>ygDG6ZzZ8T.25798542132023394.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>59</td>\n",
       "      <td>z6qfxfW6F4.4237818575016189.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>60</td>\n",
       "      <td>zu5R84I8ST.7841470681049597.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    key                            filename\n",
       "40   40     jiTmYb7pPM.7937506604855339.txt\n",
       "41   41     juE9npQetZ.5163804793372522.txt\n",
       "42   42     kr2ybHdzaR.9232864911517599.txt\n",
       "43   43    lDgbs8m1tf.08676825930990162.txt\n",
       "44   44     lR1GLF0dl3.9833018883394823.txt\n",
       "45   45    lZdgCU6d1U.12307985178981151.txt\n",
       "46   46     lsKjqDEeir.4889217778570776.txt\n",
       "47   47     m0xVYqvN8d.7758863033939478.txt\n",
       "48   48     npow2gShDJ.7019849575646654.txt\n",
       "49   49    p8IbJYSXxE.22562450196421968.txt\n",
       "50   50     rxt3ocjppG.7973298407205462.txt\n",
       "51   51    sGN3CBtN8o.35422772005643977.txt\n",
       "52   52     sHMzQdloFS.2681163681335329.txt\n",
       "53   53  tiMS4ch5zy.0033628833141092773.txt\n",
       "54   54     vNPTbjqdSJ.7197300852931762.txt\n",
       "55   55     vPSU9hwW5E.2173308100445802.txt\n",
       "56   56     wZyhwhK1Oa.6182611089275415.txt\n",
       "57   57     ySljWqQVmz.9326708723080476.txt\n",
       "58   58    ygDG6ZzZ8T.25798542132023394.txt\n",
       "59   59     z6qfxfW6F4.4237818575016189.txt\n",
       "60   60     zu5R84I8ST.7841470681049597.txt"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_key_df.columns = ['key', 'filename']\n",
    "test_key_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2ed84d4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<21x4188 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 5206 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_feature_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "3ad5e952",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jiTmYb7pPM.7937506604855339.txt\n",
      "juE9npQetZ.5163804793372522.txt\n",
      "kr2ybHdzaR.9232864911517599.txt\n",
      "lDgbs8m1tf.08676825930990162.txt\n",
      "lR1GLF0dl3.9833018883394823.txt\n",
      "lZdgCU6d1U.12307985178981151.txt\n",
      "lsKjqDEeir.4889217778570776.txt\n",
      "m0xVYqvN8d.7758863033939478.txt\n",
      "npow2gShDJ.7019849575646654.txt\n",
      "p8IbJYSXxE.22562450196421968.txt\n",
      "rxt3ocjppG.7973298407205462.txt\n",
      "sGN3CBtN8o.35422772005643977.txt\n",
      "sHMzQdloFS.2681163681335329.txt\n",
      "tiMS4ch5zy.0033628833141092773.txt\n",
      "vNPTbjqdSJ.7197300852931762.txt\n",
      "vPSU9hwW5E.2173308100445802.txt\n",
      "wZyhwhK1Oa.6182611089275415.txt\n",
      "ySljWqQVmz.9326708723080476.txt\n",
      "ygDG6ZzZ8T.25798542132023394.txt\n",
      "z6qfxfW6F4.4237818575016189.txt\n",
      "zu5R84I8ST.7841470681049597.txt\n"
     ]
    }
   ],
   "source": [
    "# Apen\n",
    "for index, row in test_key_df.iterrows():\n",
    "    key = int(row['key']) - 39\n",
    "    \n",
    "    #print(index , key)\n",
    "    print(row['filename'])\n",
    "#     test_key_df.loc[test_key_df.index[key], 'normalized_text'] = normalized_corpus[key + 39]\n",
    "#     test_key_df.loc[test_key_df.index[key], 'predicted_group'] = km_obj.predict(tfidf_feature_matrix[key])[0]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "9989e2c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>filename</th>\n",
       "      <th>normalized_text</th>\n",
       "      <th>predicted_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>40</td>\n",
       "      <td>jiTmYb7pPM.7937506604855339.txt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>41</td>\n",
       "      <td>juE9npQetZ.5163804793372522.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>kr2ybHdzaR.9232864911517599.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>43</td>\n",
       "      <td>lDgbs8m1tf.08676825930990162.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>lR1GLF0dl3.9833018883394823.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>45</td>\n",
       "      <td>lZdgCU6d1U.12307985178981151.txt</td>\n",
       "      <td>News News Local News Crime Public Safety Your ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>46</td>\n",
       "      <td>lsKjqDEeir.4889217778570776.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>47</td>\n",
       "      <td>m0xVYqvN8d.7758863033939478.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>48</td>\n",
       "      <td>npow2gShDJ.7019849575646654.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>49</td>\n",
       "      <td>p8IbJYSXxE.22562450196421968.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>50</td>\n",
       "      <td>rxt3ocjppG.7973298407205462.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>51</td>\n",
       "      <td>sGN3CBtN8o.35422772005643977.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>52</td>\n",
       "      <td>sHMzQdloFS.2681163681335329.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>53</td>\n",
       "      <td>tiMS4ch5zy.0033628833141092773.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54</td>\n",
       "      <td>vNPTbjqdSJ.7197300852931762.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>55</td>\n",
       "      <td>vPSU9hwW5E.2173308100445802.txt</td>\n",
       "      <td>NRL AFL Cricket NBA Boxing MMA Football Rugby ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>wZyhwhK1Oa.6182611089275415.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>57</td>\n",
       "      <td>ySljWqQVmz.9326708723080476.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>58</td>\n",
       "      <td>ygDG6ZzZ8T.25798542132023394.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>59</td>\n",
       "      <td>z6qfxfW6F4.4237818575016189.txt</td>\n",
       "      <td>Skip to main content NFL News NFL Draft NFL Ti...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>60</td>\n",
       "      <td>zu5R84I8ST.7841470681049597.txt</td>\n",
       "      <td>JavaScript is not available We ve detected tha...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    key                            filename  \\\n",
       "40   40     jiTmYb7pPM.7937506604855339.txt   \n",
       "41   41     juE9npQetZ.5163804793372522.txt   \n",
       "42   42     kr2ybHdzaR.9232864911517599.txt   \n",
       "43   43    lDgbs8m1tf.08676825930990162.txt   \n",
       "44   44     lR1GLF0dl3.9833018883394823.txt   \n",
       "45   45    lZdgCU6d1U.12307985178981151.txt   \n",
       "46   46     lsKjqDEeir.4889217778570776.txt   \n",
       "47   47     m0xVYqvN8d.7758863033939478.txt   \n",
       "48   48     npow2gShDJ.7019849575646654.txt   \n",
       "49   49    p8IbJYSXxE.22562450196421968.txt   \n",
       "50   50     rxt3ocjppG.7973298407205462.txt   \n",
       "51   51    sGN3CBtN8o.35422772005643977.txt   \n",
       "52   52     sHMzQdloFS.2681163681335329.txt   \n",
       "53   53  tiMS4ch5zy.0033628833141092773.txt   \n",
       "54   54     vNPTbjqdSJ.7197300852931762.txt   \n",
       "55   55     vPSU9hwW5E.2173308100445802.txt   \n",
       "56   56     wZyhwhK1Oa.6182611089275415.txt   \n",
       "57   57     ySljWqQVmz.9326708723080476.txt   \n",
       "58   58    ygDG6ZzZ8T.25798542132023394.txt   \n",
       "59   59     z6qfxfW6F4.4237818575016189.txt   \n",
       "60   60     zu5R84I8ST.7841470681049597.txt   \n",
       "\n",
       "                                      normalized_text  predicted_group  \n",
       "40                                                NaN              NaN  \n",
       "41  JavaScript is not available We ve detected tha...              0.0  \n",
       "42  JavaScript is not available We ve detected tha...              0.0  \n",
       "43  JavaScript is not available We ve detected tha...              0.0  \n",
       "44  JavaScript is not available We ve detected tha...              1.0  \n",
       "45  News News Local News Crime Public Safety Your ...              1.0  \n",
       "46  JavaScript is not available We ve detected tha...              0.0  \n",
       "47  Skip to main content NFL News NFL Draft NFL Ti...              1.0  \n",
       "48  JavaScript is not available We ve detected tha...              0.0  \n",
       "49  JavaScript is not available We ve detected tha...              0.0  \n",
       "50  Skip to main content NFL News NFL Draft NFL Ti...              1.0  \n",
       "51  Skip to main content NFL News NFL Draft NFL Ti...              0.0  \n",
       "52  JavaScript is not available We ve detected tha...              1.0  \n",
       "53  JavaScript is not available We ve detected tha...              0.0  \n",
       "54  Skip to main content NFL News NFL Draft NFL Ti...              0.0  \n",
       "55  NRL AFL Cricket NBA Boxing MMA Football Rugby ...              1.0  \n",
       "56  Skip to main content NFL News NFL Draft NFL Ti...              1.0  \n",
       "57  JavaScript is not available We ve detected tha...              0.0  \n",
       "58  JavaScript is not available We ve detected tha...              0.0  \n",
       "59  Skip to main content NFL News NFL Draft NFL Ti...              0.0  \n",
       "60  JavaScript is not available We ve detected tha...              0.0  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_key_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "f469257e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(key_df[1:2]['normalized_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "ad87d3ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "key_df.to_csv('test_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40df4d05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80ec5490",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus.reader.plaintext import PlaintextCorpusReader\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "3de09850",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpuspath = 'D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\unclean_text_data_test\\\\'\n",
    "corpus = PlaintextCorpusReader(corpuspath, '.*' , )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8f98f3a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_list = list(set(corpus.fileids())) # get unique file list (This dodent apply in real data files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c6fd81cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "85090fbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "test_file_name_list = list(set(random.choices(files_list, k=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "fdb0d64d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for filename in test_list:\n",
    "    print(filename)\n",
    "    train_list.remove(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "a88e583c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/lR1GLF0dl3.9833018883394823.txt\n",
      "1/Jr1jDrrbqy.19224090056998133.txt\n",
      "2/h8deBWDHFu.15330512422735665.txt\n",
      "2/MxfyEFwodh.6291880775265848.txt\n",
      "3/kr2ybHdzaR.9232864911517599.txt\n",
      "3/Zou9I67X0r.22250236429049453.txt\n",
      "3/lDgbs8m1tf.08676825930990162.txt\n",
      "1/AshZjMudhh.05641859642807312.txt\n",
      "3/99xzhtXoQO.8203288815364953.txt\n",
      "3/juE9npQetZ.5163804793372522.txt\n",
      "3/3B9wLEPpb4.11331682998371562.txt\n",
      "3/DhR4Azo32a.4717029879255846.txt\n",
      "3/3KF0FrO05V.7702580431456981.txt\n",
      "3/Biq3eTMrTJ.9033166229586829.txt\n",
      "3/RkhD6dTdiE.6111538098809065.txt\n",
      "2/rxt3ocjppG.7973298407205462.txt\n",
      "1/R0IAlteJhG.7059206210470494.txt\n",
      "1/g0uT3UYPDH.787621649546468.txt\n"
     ]
    }
   ],
   "source": [
    "traintrain_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "af9f75f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "c4a2bc18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_file_name_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "883df3c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "testFileNameDF = pd.DataFrame(list(dict.fromkeys(test_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "f28fd713",
   "metadata": {},
   "outputs": [],
   "source": [
    "#testFileNameDF.to_csv('D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\\\datasets\\\\test_file_names.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "88d8ea39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_file_names = pd.read_csv('D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\\\datasets\\\\test_file_names.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "e31447e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mannualy Copy files from clean_text_data dir to final_train_data dir\n",
    "# Removing randomly generated files from final_train_data to final_test_data dir\n",
    "# file_source = 'D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\datasets\\\\final_train_data\\\\'\n",
    "file_source = 'D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\unclean_text_data_test\\\\'\n",
    "file_destination = 'D:\\\\Personal\\\\Msc\\\\CMM706 Text Analytics\\\\Coursework\\\\datasets\\\\final_test_data\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ee593686",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_name_list = test_file_names['0'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a4adfcf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "import os\n",
    "for file in test_file_name_list:\n",
    "    shutil.move(file_source + file, file_destination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ce0378d",
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeanModel, clusters = k_means(feature_matrix=train_feature_matrix, num_clusters=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "bf5c0b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc8ac0e6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}