classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1) (60000,)\n",
      "(10000, 28, 28, 1) (10000,)\n"
     ]
    }
   ],
   "source": [
    "dataset_dir = \"./mnist_data/MNIST/raw/\"\n",
    "\n",
    "train_features, train_labels = mnist.load_dataset(\n",
    "    os.path.join(dataset_dir, \"train-images-idx3-ubyte.gz\"),\n",
    "    os.path.join(dataset_dir, \"train-labels-idx1-ubyte.gz\"),\n",
    "    total_images=60000,\n",
    "    image_size=28,\n",
    ")\n",
    "train_features /= 255.0\n",
    "\n",
    "test_features, test_labels = mnist.load_dataset(\n",
    "    os.path.join(dataset_dir, \"t10k-images-idx3-ubyte.gz\"),\n",
    "    os.path.join(dataset_dir, \"t10k-labels-idx1-ubyte.gz\"),\n",
    "    total_images=10000,\n",
    "    image_size=28,\n",
    ")\n",
    "test_features /= 255.0\n",
    "\n",
    "print(train_features.shape, train_labels.shape)\n",
    "print(test_features.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGxCAYAAADLfglZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgHUlEQVR4nO3df3RU9bnv8c+QhBFpMkfEJBOIMSLUSigqIj9EDVCziEtEU09RT1uoP4o10EUDUiltjViJRyuL9lKhVRulgrDsRaCFpcZCgi6gBopHLlUaLqGEIzGXiJkYYCBh3z+4zO2YENzDTJ5M8n6ttddy9t7P7CebLR++mT3f7XEcxxEAAAZ6WDcAAOi+CCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIXRLL730kjwej7Zv3x6V9/N4PJo+fXpU3utf37O4uDii2h07dqiwsFBDhgxRcnKy0tLS9I1vfEMbN26Mao/A+SKEgC7o1Vdf1Xvvvaf77rtPa9eu1QsvvCCv16vx48dr2bJl1u0BIYnWDQCIvjlz5uiXv/xl2Lpbb71V1157rebPn6/vfve7Rp0B4RgJAWdx/PhxzZo1S1dffbV8Pp/69OmjUaNGae3atWet+e1vf6tBgwbJ6/Xqqquu0sqVK1vtU1tbq2nTpql///7q2bOnsrOz9fjjj6u5uTlqvaemprZal5CQoGHDhqmmpiZqxwHOFyMh4CyCwaA+/fRTzZ49W/369dOJEyf09ttvq6CgQKWlpa1GE+vWrdOmTZs0f/589e7dW88995zuueceJSYm6q677pJ0OoCuv/569ejRQz//+c81YMAAbd26Vb/4xS+0f/9+lZaWttvTZZddJknav3+/65+nublZ77zzjgYPHuy6FogVQgg4C5/PFxYKLS0tGj9+vI4cOaJFixa1CqHDhw+rsrJSaWlpkk7/+isnJ0dz584NhVBxcbGOHDmi3bt369JLL5UkjR8/Xr169dLs2bP1yCOP6KqrrjprT4mJkf8vW1xcrL1792rNmjURvwcQbfw6DmjHa6+9phtuuEFf+cpXlJiYqKSkJL344ov68MMPW+07fvz4UABJp3/9NXnyZO3du1cHDx6UJP35z3/W2LFjlZGRoebm5tCSn58vSaqoqGi3n71792rv3r2uf44XXnhBTz75pGbNmqVJkya5rgdihRACzmL16tX61re+pX79+umVV17R1q1bVVlZqfvuu0/Hjx9vtX96evpZ19XX10uSPvnkE/3pT39SUlJS2HLmV2SHDx+O+s9RWlqqadOm6fvf/76eeeaZqL8/cD74dRxwFq+88oqys7O1atUqeTye0PpgMNjm/rW1tWddd/HFF0uS+vbtq69//et68skn23yPjIyM8207TGlpqR544AFNmTJFS5cuDfs5gM6AEALOwuPxqGfPnmF/cdfW1p717ri//OUv+uSTT0K/kmtpadGqVas0YMAA9e/fX5J02223acOGDRowYIAuuuiimPb/0ksv6YEHHtC3v/1tvfDCCwQQOiVCCN3axo0b27zT7NZbb9Vtt92m1atX6+GHH9Zdd92lmpoaPfHEE/L7/aqqqmpV07dvX40bN04/+9nPQnfHffTRR2G3ac+fP19lZWUaPXq0fvjDH+qrX/2qjh8/rv3792vDhg1aunRpKLDacsUVV0jSOT8Xeu2113T//ffr6quv1rRp0/Tee++Fbb/mmmvk9XrbfQ+gIxBC6NZ+/OMft7m+urpa3/ve91RXV6elS5fq97//vS6//HI9+uijOnjwoB5//PFWNbfffrsGDx6sn/70pzpw4IAGDBig5cuXa/LkyaF9/H6/tm/frieeeELPPPOMDh48qOTkZGVnZ2vChAnnHB192e8SrV+/XqdOndLf/vY33XDDDW3+fGdu9wYseRzHcaybAAB0T9wdBwAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMdLrvCZ06dUoff/yxkpOT+YY3AMQhx3HU2NiojIwM9ejR/lin04XQxx9/rMzMTOs2AADnqaampt0ZQKROGELJycmSpDG6VYlKMu4GAOBWs07qXW0I/X3enpiF0HPPPadnnnlGhw4d0uDBg7Vo0SLdeOON56w78yu4RCUp0UMIAUDc+X/z8HyZj1RicmPCqlWrNHPmTM2bN087d+7UjTfeqPz8fB04cCAWhwMAxKmYhNDChQt1//3364EHHtDXvvY1LVq0SJmZmVqyZEksDgcAiFNRD6ETJ05ox44dysvLC1ufl5enLVu2tNo/GAwqEAiELQCA7iHqIXT48GG1tLSEHux1RlpaWptPniwpKZHP5wst3BkHAN1HzL6s+sUPpBzHafNDqrlz56qhoSG01NTUxKolAEAnE/W74/r27auEhIRWo566urpWoyNJ8nq9POERALqpqI+EevbsqWHDhqmsrCxs/ZlHGgMAcEZMvidUVFSk73znO7ruuus0atQo/e53v9OBAwf00EMPxeJwAIA4FZMQmjx5surr6zV//nwdOnRIOTk52rBhg7KysmJxOABAnPI4juNYN/GvAoGAfD6fcjWJGRMAIA41OydVrrVqaGhQSkpKu/vyKAcAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZhKtGwDw5SRc3Md1jceXEtGxDnwzw3XN8b6O65orHv8v1zWnjh51XYPOi5EQAMAMIQQAMBP1ECouLpbH4wlb0tPTo30YAEAXEJPPhAYPHqy333479DohISEWhwEAxLmYhFBiYiKjHwDAOcXkM6GqqiplZGQoOztbd999t/bt23fWfYPBoAKBQNgCAOgeoh5CI0aM0LJly/Tmm2/q+eefV21trUaPHq36+vo29y8pKZHP5wstmZmZ0W4JANBJeRzHcX9zvwtNTU0aMGCA5syZo6Kiolbbg8GggsFg6HUgEFBmZqZyNUmJnqRYtgbEFb4ndBrfE+r8mp2TKtdaNTQ0KCWl/Wsw5l9W7d27t4YMGaKqqqo2t3u9Xnm93li3AQDohGL+PaFgMKgPP/xQfr8/1ocCAMSZqIfQ7NmzVVFRoerqav31r3/VXXfdpUAgoClTpkT7UACAOBf1X8cdPHhQ99xzjw4fPqxLLrlEI0eO1LZt25SVlRXtQwEA4lzUQ2jlypXRfkugU+uRc6Xrmqq5vVzX3Ddki+uaWRe/6bqmI30t7SHXNQOn7ohBJ7DC3HEAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMxPyhdoAFz/AhEdXt/VGC65ryMYtd11yS4P5Bjj0i+Dfj+qMXua6RpH3BVNc1hRftcV3zh5ued13zxHD3j4VxKne5rkHHYCQEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADDDLNroUAmXXOK65h+/6ue65k+jn3NdI0mXJyVFUOV+RuxIlAYyXdes+eaYiI51yuv+PBT+2f0s2td5W1zXHEvr5brmAtcV6CiMhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJhhAlN0qP/+9kDXNbtv/lUER4pkItKO80okk5HeMdp1Tcuef7iukSTPNYMjqgPcYiQEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADBOYokP1u32/dQvt+uPn6a5rFv5jvOuatDmO65qWPVWuayJ1ZEhKhx0L3RsjIQCAGUIIAGDGdQht3rxZEydOVEZGhjwej9asWRO23XEcFRcXKyMjQ7169VJubq52794drX4BAF2I6xBqamrS0KFDtXjx4ja3P/3001q4cKEWL16syspKpaen65ZbblFjY+N5NwsA6Fpc35iQn5+v/Pz8Nrc5jqNFixZp3rx5KigokCS9/PLLSktL04oVKzRt2rTz6xYA0KVE9TOh6upq1dbWKi8vL7TO6/Xq5ptv1pYtW9qsCQaDCgQCYQsAoHuIagjV1tZKktLS0sLWp6WlhbZ9UUlJiXw+X2jJzMyMZksAgE4sJnfHeTyesNeO47Rad8bcuXPV0NAQWmpqamLREgCgE4rql1XT009/0a+2tlZ+vz+0vq6urtXo6Ayv1yuv1xvNNgAAcSKqI6Hs7Gylp6errKwstO7EiROqqKjQ6NGjo3koAEAX4Hok9Pnnn2vv3r2h19XV1Xr//ffVp08fXXrppZo5c6YWLFiggQMHauDAgVqwYIEuvPBC3XvvvVFtHAAQ/1yH0Pbt2zV27NjQ66KiIknSlClT9NJLL2nOnDk6duyYHn74YR05ckQjRozQW2+9peTk5Oh1DQDoElyHUG5urhzn7JMvejweFRcXq7i4+Hz6Qlf1oPvP/64qnOG6JrOsxXWNJPXe3fZdnO3p+89/uK6JrLuOczSt7RuJgGhj7jgAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgJmoPlkVOJeWvdWua674kfuaSDV32JE6t5PDG61bQDfBSAgAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZJjAFztOBn492XdN8oeP+QB73JYrgMJJUMHBrZIUuTT+Y67qm1xt/c10T4WlAB2AkBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwTmKLTS0hJcV1z/PqBER0rae4nrms+uPJ/RHQst5I8Ca5rTjotMeikbZuOXei65uD3L3Vd4zR/6LoGnRcjIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGaYwBQR83i9rmtO3DzEdc2PnvuD65qxvf7iukaSPmkJuq7ZdOwi1zU//8ck1zWvDn7JdU1Govs/o0hd0OOk65p93/o31zWX77nAdc2p48dd16BjMBICAJghhAAAZlyH0ObNmzVx4kRlZGTI4/FozZo1YdunTp0qj8cTtowcOTJa/QIAuhDXIdTU1KShQ4dq8eLFZ91nwoQJOnToUGjZsGHDeTUJAOiaXN+YkJ+fr/z8/Hb38Xq9Sk9Pj7gpAED3EJPPhMrLy5WamqpBgwbpwQcfVF1d3Vn3DQaDCgQCYQsAoHuIegjl5+dr+fLl2rhxo5599llVVlZq3LhxCgbbvvW1pKREPp8vtGRmZka7JQBAJxX17wlNnjw59N85OTm67rrrlJWVpfXr16ugoKDV/nPnzlVRUVHodSAQIIgAoJuI+ZdV/X6/srKyVFVV1eZ2r9crbwRfegQAxL+Yf0+ovr5eNTU18vv9sT4UACDOuB4Jff7559q7d2/odXV1td5//3316dNHffr0UXFxsb75zW/K7/dr//79+slPfqK+ffvqzjvvjGrjAID45zqEtm/frrFjx4Zen/k8Z8qUKVqyZIl27dqlZcuW6bPPPpPf79fYsWO1atUqJScnR69rAECX4HEcx7Fu4l8FAgH5fD7lapISPUnW7XQLPS5wPyGkJNVPvsZ1zTsLfh3Rsdwa/OqMiOr6b2pxXeNdX+m6JtHv/nt0N7xZ7bpm1sX/y3VNZzfqiR+6rklb9l8RHevU0aMR1XV3zc5JlWutGhoalJKS0u6+zB0HADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADAT8yeromN5InhK7UcLvx7RsT6a1DEzYk/ac4frmkHP7IvoWC2f1LmuSczs77pm6LoDrmseufjvrmsaTp1wXSNJI/7nLNc1/ivdn7u/DFnlumbrz9xfd5Pvuc11jSQd/vUQ1zUX1J+M6FhuJZT/rUOOE2uMhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJhhAtNOzJPo/o9nz6Khrms+uv03rmsk6WBz0HXN7b+d47rmst//b9c1zRFMRCpJJ78xzHVNzn/udF3zWOoO1zWlgSzXNX+YN9F1jSRdsXqb65qEvhe7rsm9ZYbrmqbJDa5rXr/medc1ktT/1+4nBI7En5vcn7vfDbo8Bp10PEZCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzDCBaSdW88j1rms+uv1Xrms+jmAiUkn696cecV1z2Zp9rms+HZftusb5drLrGkn6Y47783dJgvtJLgevdD9x56DfHXZdc+Gev7quiVTL4XrXNSmvRlLjukR3Pex+4lxJSrvrnxHVuTbr3yIo2h3tLkwwEgIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGDG4ziOY93EvwoEAvL5fMrVJCV6kqzbMTVv3/uua0Z4T7qu+bQlsglMlx4Z4bqmX88jrmumpHTQJJIRGrzih65rrphb6brGaW52XQNYaHZOqlxr1dDQoJSUlHb3ZSQEADBDCAEAzLgKoZKSEg0fPlzJyclKTU3VHXfcoT179oTt4ziOiouLlZGRoV69eik3N1e7d3eN514AAKLLVQhVVFSosLBQ27ZtU1lZmZqbm5WXl6empqbQPk8//bQWLlyoxYsXq7KyUunp6brlllvU2NgY9eYBAPHN1ZNV33jjjbDXpaWlSk1N1Y4dO3TTTTfJcRwtWrRI8+bNU0FBgSTp5ZdfVlpamlasWKFp06ZFr3MAQNw7r8+EGhoaJEl9+vSRJFVXV6u2tlZ5eXmhfbxer26++WZt2bKlzfcIBoMKBAJhCwCge4g4hBzHUVFRkcaMGaOcnBxJUm1trSQpLS0tbN+0tLTQti8qKSmRz+cLLZmZmZG2BACIMxGH0PTp0/XBBx/o1VdfbbXN4/GEvXYcp9W6M+bOnauGhobQUlNTE2lLAIA44+ozoTNmzJihdevWafPmzerfv39ofXp6uqTTIyK/3x9aX1dX12p0dIbX65XX642kDQBAnHM1EnIcR9OnT9fq1au1ceNGZWdnh23Pzs5Wenq6ysrKQutOnDihiooKjR49OjodAwC6DFcjocLCQq1YsUJr165VcnJy6HMen8+nXr16yePxaObMmVqwYIEGDhyogQMHasGCBbrwwgt17733xuQHAADEL1chtGTJEklSbm5u2PrS0lJNnTpVkjRnzhwdO3ZMDz/8sI4cOaIRI0borbfeUnJyclQaBgB0HUxg2ond+MFx1zWPXLwrBp3Yuu2jAtc1B7b2P/dObbj8jw2ua5zde93XnDzhugaIF0xgCgCIC4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAMxE9WRUdY8vYDNc1I/5jnOuahqGRzeic+H/cz3I+aOl/uz9ObZ3rmsuOR/aY+FMRVQGIFCMhAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZpjAtBNrqf/UdU3ar7e4r3FdEbnmDjwWgM6PkRAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM65CqKSkRMOHD1dycrJSU1N1xx13aM+ePWH7TJ06VR6PJ2wZOXJkVJsGAHQNrkKooqJChYWF2rZtm8rKytTc3Ky8vDw1NTWF7TdhwgQdOnQotGzYsCGqTQMAuoZENzu/8cYbYa9LS0uVmpqqHTt26Kabbgqt93q9Sk9Pj06HAIAu67w+E2poaJAk9enTJ2x9eXm5UlNTNWjQID344IOqq6s763sEg0EFAoGwBQDQPUQcQo7jqKioSGPGjFFOTk5ofX5+vpYvX66NGzfq2WefVWVlpcaNG6dgMNjm+5SUlMjn84WWzMzMSFsCAMQZj+M4TiSFhYWFWr9+vd59913179//rPsdOnRIWVlZWrlypQoKClptDwaDYQEVCASUmZmpXE1SoicpktYAAIaanZMq11o1NDQoJSWl3X1dfSZ0xowZM7Ru3Tpt3ry53QCSJL/fr6ysLFVVVbW53ev1yuv1RtIGACDOuQohx3E0Y8YMvf766yovL1d2dvY5a+rr61VTUyO/3x9xkwCArsnVZ0KFhYV65ZVXtGLFCiUnJ6u2tla1tbU6duyYJOnzzz/X7NmztXXrVu3fv1/l5eWaOHGi+vbtqzvvvDMmPwAAIH65GgktWbJEkpSbmxu2vrS0VFOnTlVCQoJ27dqlZcuW6bPPPpPf79fYsWO1atUqJScnR61pAEDX4PrXce3p1auX3nzzzfNqCADQfTB3HADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADATKJ1A1/kOI4kqVknJce4GQCAa806Ken//33enk4XQo2NjZKkd7XBuBMAwPlobGyUz+drdx+P82WiqgOdOnVKH3/8sZKTk+XxeMK2BQIBZWZmqqamRikpKUYd2uM8nMZ5OI3zcBrn4bTOcB4cx1FjY6MyMjLUo0f7n/p0upFQjx491L9//3b3SUlJ6dYX2Rmch9M4D6dxHk7jPJxmfR7ONQI6gxsTAABmCCEAgJm4CiGv16vHHntMXq/XuhVTnIfTOA+ncR5O4zycFm/nodPdmAAA6D7iaiQEAOhaCCEAgBlCCABghhACAJghhAAAZuIqhJ577jllZ2frggsu0LBhw/TOO+9Yt9ShiouL5fF4wpb09HTrtmJu8+bNmjhxojIyMuTxeLRmzZqw7Y7jqLi4WBkZGerVq5dyc3O1e/dum2Zj6FznYerUqa2uj5EjR9o0GyMlJSUaPny4kpOTlZqaqjvuuEN79uwJ26c7XA9f5jzEy/UQNyG0atUqzZw5U/PmzdPOnTt14403Kj8/XwcOHLBurUMNHjxYhw4dCi27du2ybinmmpqaNHToUC1evLjN7U8//bQWLlyoxYsXq7KyUunp6brllltCk+F2Fec6D5I0YcKEsOtjw4auNRFwRUWFCgsLtW3bNpWVlam5uVl5eXlqamoK7dMdrocvcx6kOLkenDhx/fXXOw899FDYuiuvvNJ59NFHjTrqeI899pgzdOhQ6zZMSXJef/310OtTp0456enpzlNPPRVad/z4ccfn8zlLly416LBjfPE8OI7jTJkyxZk0aZJJP1bq6uocSU5FRYXjON33evjieXCc+Lke4mIkdOLECe3YsUN5eXlh6/Py8rRlyxajrmxUVVUpIyND2dnZuvvuu7Vv3z7rlkxVV1ertrY27Nrwer26+eabu921IUnl5eVKTU3VoEGD9OCDD6qurs66pZhqaGiQJPXp00dS970evngezoiH6yEuQujw4cNqaWlRWlpa2Pq0tDTV1tYaddXxRowYoWXLlunNN9/U888/r9raWo0ePVr19fXWrZk58+ff3a8NScrPz9fy5cu1ceNGPfvss6qsrNS4ceMUDAatW4sJx3FUVFSkMWPGKCcnR1L3vB7aOg9S/FwPne5RDu354vOFHMdpta4ry8/PD/33kCFDNGrUKA0YMEAvv/yyioqKDDuz192vDUmaPHly6L9zcnJ03XXXKSsrS+vXr1dBQYFhZ7Exffp0ffDBB3r33XdbbetO18PZzkO8XA9xMRLq27evEhISWv1Lpq6urtW/eLqT3r17a8iQIaqqqrJuxcyZuwO5Nlrz+/3KysrqktfHjBkztG7dOm3atCns+WPd7Xo423loS2e9HuIihHr27Klhw4aprKwsbH1ZWZlGjx5t1JW9YDCoDz/8UH6/37oVM9nZ2UpPTw+7Nk6cOKGKiopufW1IUn19vWpqarrU9eE4jqZPn67Vq1dr48aNys7ODtveXa6Hc52HtnTa68HwpghXVq5c6SQlJTkvvvii8/e//92ZOXOm07t3b2f//v3WrXWYWbNmOeXl5c6+ffucbdu2ObfddpuTnJzc5c9BY2Ojs3PnTmfnzp2OJGfhwoXOzp07nX/+85+O4zjOU0895fh8Pmf16tXOrl27nHvuucfx+/1OIBAw7jy62jsPjY2NzqxZs5wtW7Y41dXVzqZNm5xRo0Y5/fr161Ln4Qc/+IHj8/mc8vJy59ChQ6Hl6NGjoX26w/VwrvMQT9dD3ISQ4zjOb37zGycrK8vp2bOnc+2114bdjtgdTJ482fH7/U5SUpKTkZHhFBQUOLt377ZuK+Y2bdrkSGq1TJkyxXGc07flPvbYY056errj9Xqdm266ydm1a5dt0zHQ3nk4evSok5eX51xyySVOUlKSc+mllzpTpkxxDhw4YN12VLX180tySktLQ/t0h+vhXOchnq4HnicEADATF58JAQC6JkIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCY+b/7qZ1bYzzccwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the image\n",
    "idx = 5\n",
    "image = train_features[idx]\n",
    "plt.title(f\"Label: {train_labels[idx]}\")\n",
    "plt.imshow(image)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = train_features.reshape(train_features.shape[0], -1)\n",
    "test_features = test_features.reshape(test_features.shape[0], -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "\n",
    "## 1. What Problem is Logistic Regression Used to Solve?\n",
    "\n",
    "Logistic regression is used to solve **classification problems**, where the goal is to predict discrete categories or labels rather than continuous values. While standard logistic regression is often used for **binary classification** (predicting one of two possible classes, like “spam” or “not spam”), the approach can also be extended to handle **multiclass classification** tasks. This extension is called **multinomial logistic regression** (or softmax regression). Multinomial logistic regression allows us to classify data into more than two categories, such as predicting types of animals, customer segments, or product categories.\n",
    "\n",
    "## 2. What Assumption Does the Model Make?\n",
    "\n",
    "Logistic regression assumes a **linear relationship** between the input features and the **log-odds** (logarithm of the odds) of the class probabilities. This assumption implies that a linear combination of the input features can adequately separate the classes. For binary logistic regression, this means the log-odds of the probability of one class versus the other is linearly related to the features.\n",
    "\n",
    "In multinomial logistic regression, we extend this assumption by creating separate linear functions for each class. For $K$ classes, we have $K$ sets of weights and intercepts, each modeling the log-odds of one class relative to a baseline class. This results in $K - 1$ decision boundaries in the feature space, each approximated as a linear function, dividing the space into regions corresponding to each class.\n",
    "\n",
    "## 3. What is the Model?\n",
    "\n",
    "Logistic regression uses a **logistic function** (sigmoid) to transform the linear combination of input features into a probability between 0 and 1. The model for binary logistic regression is:\n",
    "\n",
    "$$\n",
    "P(y=1|X) = \\sigma(z) = \\frac{1}{1 + e^{-z}}\n",
    "$$\n",
    "\n",
    "where $z = \\beta_0 + \\beta_1 x_1 + \\beta_2 x_2 + ... + \\beta_n x_n$, and $\\sigma(z)$ is the sigmoid function. Here, $P(y=1|X)$ represents the probability that the input $X$ belongs to the positive class.\n",
    "\n",
    "For **multinomial logistic regression**, we generalize this with the **softmax function**. Given $K$ possible classes, the model computes the probability for each class $j$ as:\n",
    "\n",
    "$$\n",
    "P(y=j|X) = \\frac{e^{z_j}}{\\sum_{k=1}^K e^{z_k}}\n",
    "$$\n",
    "\n",
    "where $z_j = \\beta_{j0} + \\beta_{j1} x_1 + \\beta_{j2} x_2 + ... + \\beta_{jn} x_n$ is the linear score for class $j$. The softmax function ensures that the probabilities for all classes sum to 1, making it suitable for multiclass classification. The class with the highest probability is typically chosen as the predicted class.\n",
    "\n",
    "## 4. How is the Model Trained?\n",
    "\n",
    "Logistic regression is trained by **maximizing the likelihood** of the observed class labels, a process known as **maximum likelihood estimation (MLE)**. In binary logistic regression, this involves minimizing a **log loss** (or binary cross-entropy) function:\n",
    "\n",
    "$$\n",
    "\\text{Log Loss} = -\\frac{1}{N} \\sum_{i=1}^N \\left[ y_i \\log(\\hat{y}_i) + (1 - y_i) \\log(1 - \\hat{y}_i) \\right]\n",
    "$$\n",
    "\n",
    "where $\\hat{y}_i$ is the predicted probability for the positive class for each data point $i$, and $y_i$ is the actual label.\n",
    "\n",
    "For **multinomial logistic regression**, the model optimizes a **multiclass log loss** (or cross-entropy) function:\n",
    "\n",
    "$$\n",
    "\\text{Multiclass Log Loss} = -\\frac{1}{N} \\sum_{i=1}^N \\sum_{j=1}^K y_{ij} \\log(P(y=j|X_i))\n",
    "$$\n",
    "\n",
    "where $y_{ij}$ is a binary indicator (1 if sample $i$ belongs to class $j$, 0 otherwise), and $P(y=j|X_i)$ is the predicted probability for class $j$.\n",
    "\n",
    "Training both binary and multinomial logistic regression models involves adjusting the weights $\\beta$ to minimize the log loss. Optimization techniques like **gradient descent** or **stochastic gradient descent** are often used to efficiently update the weights during training. This iterative process continues until the model reaches a set of weights that minimize the loss, resulting in a model that can make accurate classifications on unseen data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 0.9346166666666667\n",
      "Test accuracy: 0.9256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ericduong/miniforge3/envs/torch/lib/python3.12/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "logistic_regressor = LogisticRegression()\n",
    "logistic_regressor.fit(train_features, train_labels)\n",
    "\n",
    "logistic_reg_train_accuracy = logistic_regressor.score(train_features, train_labels)\n",
    "logistic_reg_test_accuracy = logistic_regressor.score(test_features, test_labels)\n",
    "\n",
    "print(f\"Train accuracy: {logistic_reg_train_accuracy}\")\n",
    "print(f\"Test accuracy: {logistic_reg_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-Nearest Neighbors (KNN) Algorithm\n",
    "\n",
    "## 1. What problem is KNN used to solve?\n",
    "K-Nearest Neighbors (KNN) is a supervised machine learning algorithm primarily used for classification and regression tasks. It is especially popular for **multi-class classification problems** where the goal is to predict a class label for a given input based on its similarity to known instances. In a multi-class classification context, KNN assigns an input to one of several classes by looking at the class labels of its \"neighbors\" in the feature space.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "KNN assumes that **similar instances in the feature space belong to the same class**. This assumption, often referred to as the **continuity assumption**, implies that data points close in the feature space are likely to share the same class label. It works well when the decision boundaries between classes are not complex and the classes are relatively well-separated in the feature space. \n",
    "\n",
    "## 3. What is the model?\n",
    "KNN is a **non-parametric, instance-based learning model**. It does not involve any training of parameters, as it stores the entire training dataset for decision-making. For a given input point, KNN computes the distance between this point and all points in the training set, identifying the $k$ nearest points (neighbors). The **class of the input point** is determined by a majority vote of these $k$ nearest neighbors.\n",
    "\n",
    "Mathematically:\n",
    "- Let $X = \\{x_1, x_2, \\ldots, x_n\\}$ represent the feature space of $n$ training samples.\n",
    "- For a new input $x'$, KNN finds the set of $k$ points $N(x') \\subset X$ that are closest to $x'$ based on a distance metric (e.g., Euclidean distance).\n",
    "\n",
    "If $y_i$ represents the class label of a point $x_i$, the predicted class $\\hat{y}'$ of $x'$ is determined by:\n",
    "$$\n",
    "\\hat{y}' = \\arg \\max_{y} \\sum_{x_i \\in N(x')} \\mathbb{1}(y_i = y)\n",
    "$$\n",
    "where $\\mathbb{1}$ is an indicator function that equals 1 if $y_i = y$ and 0 otherwise.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "KNN is an **instance-based model**, meaning it has no explicit training phase where model parameters are optimized. Instead, **training** in KNN simply involves storing the labeled dataset. During inference or prediction, the model calculates distances between a new input and each point in the training set to find the $k$ nearest neighbors.\n",
    "\n",
    "For multi-class classification:\n",
    "1. For a new input point, compute distances to all training samples.\n",
    "2. Select the $k$ nearest samples.\n",
    "3. Assign the class that is most frequent among these $k$ neighbors.\n",
    "\n",
    "Thus, KNN's complexity lies in the prediction phase, where it needs to compare each new input with the entire training set, making it computationally intensive for large datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN Train accuracy: 0.9819166666666667\n",
      "KNN Test accuracy: 0.9688\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# Initialize the KNN model with 5 number of neighbors\n",
    "knn = KNeighborsClassifier(n_neighbors=5)\n",
    "\n",
    "# Train the KNN model\n",
    "knn.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the KNN model\n",
    "knn_train_accuracy = knn.score(train_features, train_labels)\n",
    "knn_test_accuracy = knn.score(test_features, test_labels)\n",
    "\n",
    "print(f\"KNN Train accuracy: {knn_train_accuracy}\")\n",
    "print(f\"KNN Test accuracy: {knn_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes Algorithm\n",
    "\n",
    "## 1. What problem is Naive Bayes used to solve?\n",
    "Naive Bayes is a family of supervised machine learning algorithms commonly used for **classification problems**, particularly effective for **multi-class classification**. It is widely applied in text classification (such as spam detection) and in cases where feature independence is a reasonable assumption. Naive Bayes is a probabilistic classifier that assigns a class label to a given instance by calculating the posterior probability for each class and choosing the class with the highest probability.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "Naive Bayes makes the **naive independence assumption**, meaning that it assumes all features (predictors) are conditionally independent given the class label. This assumption simplifies the computation of probabilities, as the joint probability of the features given the class can be factored into the product of the individual feature probabilities:\n",
    "$$\n",
    "P(x_1, x_2, \\ldots, x_n | y) = \\prod_{i=1}^n P(x_i | y)\n",
    "$$\n",
    "where $x_i$ represents a feature and $y$ represents the class label. Although this assumption is often violated in practice, Naive Bayes can still perform well, particularly with large datasets.\n",
    "\n",
    "## 3. What is the model?\n",
    "Naive Bayes is a **generative model** based on **Bayes’ theorem**. For a new instance with features $x = (x_1, x_2, \\ldots, x_n)$, Naive Bayes calculates the **posterior probability** for each class $y$ by using:\n",
    "$$\n",
    "P(y | x) = \\frac{P(y) \\cdot P(x | y)}{P(x)}\n",
    "$$\n",
    "Since we are only interested in the class that maximizes $P(y | x)$, we can ignore $P(x)$ (as it remains constant for all classes) and focus on:\n",
    "$$\n",
    "P(y | x) \\propto P(y) \\prod_{i=1}^n P(x_i | y)\n",
    "$$\n",
    "The model uses the prior probability $P(y)$ and the likelihood $P(x_i | y)$ for each feature $x_i$ to compute the posterior probability of each class. In a **multi-class classification** setting, Naive Bayes calculates this probability for each possible class and assigns the class with the highest probability.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "Training a Naive Bayes model involves estimating the probabilities needed for classification:\n",
    "1. **Prior probability $P(y)$** for each class $y$: This is estimated from the training data as the fraction of instances belonging to each class.\n",
    "2. **Likelihood $P(x_i | y)$** for each feature $x_i$ given the class $y$: This depends on the type of data and the specific Naive Bayes variant (e.g., Gaussian Naive Bayes for continuous data, Multinomial Naive Bayes for discrete data). For example:\n",
    "   - In a **Gaussian Naive Bayes** classifier (for continuous data), $P(x_i | y)$ is modeled with a Gaussian distribution, where mean and variance are estimated for each feature within each class.\n",
    "   - In a **Multinomial Naive Bayes** classifier (for text data or count features), $P(x_i | y)$ is based on the frequency of feature values within each class.\n",
    "\n",
    "During prediction, Naive Bayes uses the estimated priors and likelihoods to calculate the posterior probability for each class, allowing it to classify the input into one of multiple classes based on the highest posterior probability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gaussian Naive Bayes Train accuracy: 0.5649\n",
      "Gaussian Naive Bayes Test accuracy: 0.5558\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "# Initialize the Gaussian Naive Bayes model\n",
    "gnb = GaussianNB()\n",
    "\n",
    "# Train the Gaussian Naive Bayes model\n",
    "gnb.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the Gaussian Naive Bayes model\n",
    "gnb_train_accuracy = gnb.score(train_features, train_labels)\n",
    "gnb_test_accuracy = gnb.score(test_features, test_labels)\n",
    "\n",
    "print(f\"Gaussian Naive Bayes Train accuracy: {gnb_train_accuracy}\")\n",
    "print(f\"Gaussian Naive Bayes Test accuracy: {gnb_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree Algorithm\n",
    "\n",
    "## 1. What problem is Decision Tree used to solve?\n",
    "Decision Trees are supervised learning algorithms that can be used for both **classification and regression** tasks. They are particularly effective in **multi-class classification** problems, where they recursively split the data into subsets based on feature values, leading to a tree-like structure that makes classification decisions. Decision Trees are popular due to their interpretability and their ability to handle both categorical and continuous data.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "Decision Trees make a few implicit assumptions:\n",
    "1. **Feature splits can sufficiently separate classes**: The model assumes that one or more features will have values that enable effective splits to distinguish between classes.\n",
    "2. **Local decision boundaries**: Decision Trees assume that local, axis-aligned splits of features can approximate the decision boundary between classes, which works well when the data has regions that align with these splits.\n",
    "   \n",
    "These assumptions are often reasonable for simpler, well-separated data, but for complex data, they can lead to overfitting if the tree grows too deep.\n",
    "\n",
    "## 3. What is the model?\n",
    "A Decision Tree is a **tree-structured model** where:\n",
    "- **Nodes** represent a decision based on a feature.\n",
    "- **Edges** represent the outcome of a decision, leading to a new subset of the data.\n",
    "- **Leaves** represent a class label (in classification) or a value (in regression).\n",
    "\n",
    "The tree is built by iteratively selecting the best feature and value for splitting the data at each node. This selection is based on **impurity reduction**, with measures such as **Gini impurity** or **entropy** in classification tasks.\n",
    "\n",
    "For a feature $x$ that splits the data into subsets $D_1$ and $D_2$, the impurity reduction can be calculated using:\n",
    "$$\n",
    "\\text{Gain}(x) = \\text{Impurity}(D) - \\left( \\frac{|D_1|}{|D|} \\text{Impurity}(D_1) + \\frac{|D_2|}{|D|} \\text{Impurity}(D_2) \\right)\n",
    "$$\n",
    "where $|D|$ is the number of instances in subset $D$, and $\\text{Impurity}(D)$ could be calculated using Gini or entropy. \n",
    "\n",
    "The tree classifies an input by following the decisions from the root to a leaf, yielding a class label. In **multi-class classification**, each leaf node can represent one of the multiple classes based on the majority class of the instances in that node.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "Training a Decision Tree involves:\n",
    "1. **Choosing the best feature and value** to split the data at each node. This is done by calculating the impurity reduction for each feature split and selecting the one with the highest reduction.\n",
    "2. **Recursively splitting** the data into subsets at each new node until a stopping criterion is met (e.g., a maximum depth, a minimum number of samples per node, or no further impurity reduction).\n",
    "3. **Assigning a class label** at each leaf node based on the majority class among the instances in that node.\n",
    "\n",
    "The training process continues until the tree reaches a structure that can classify the data with reasonable accuracy, but careful tuning is needed to avoid **overfitting**. In the context of multi-class classification, the tree learns to partition the feature space so that each region corresponds to one of the multiple classes based on the majority rule at each leaf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Train accuracy: 1.0\n",
      "Decision Tree Test accuracy: 0.8791\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# Initialize the Decision Tree model\n",
    "decision_tree = DecisionTreeClassifier()\n",
    "\n",
    "# Train the Decision Tree model\n",
    "decision_tree.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the Decision Tree model\n",
    "decision_tree_train_accuracy = decision_tree.score(train_features, train_labels)\n",
    "decision_tree_test_accuracy = decision_tree.score(test_features, test_labels)\n",
    "\n",
    "print(f\"Decision Tree Train accuracy: {decision_tree_train_accuracy}\")\n",
    "print(f\"Decision Tree Test accuracy: {decision_tree_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machine (SVM) Algorithm\n",
    "\n",
    "## 1. What problem is Support Vector Machine used to solve?\n",
    "Support Vector Machines (SVM) are supervised learning algorithms primarily used for **classification** tasks, although they can also be used for regression. SVMs are well-suited for **binary and multi-class classification** problems, especially when there is a clear margin of separation between classes. The algorithm aims to find an optimal hyperplane that maximally separates instances of different classes, making it particularly effective in high-dimensional spaces.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "SVM assumes that:\n",
    "1. **Data is linearly separable** (in a transformed feature space if necessary): It assumes that a hyperplane or a set of hyperplanes can separate the classes with a maximum margin. If the data is not linearly separable in the input space, the **kernel trick** is applied to transform the data into a higher-dimensional space where it becomes separable.\n",
    "2. **Maximizing the margin improves generalization**: SVM assumes that the best decision boundary is the one that maximizes the margin (distance) between the closest points (support vectors) of different classes. This helps the model generalize well to unseen data.\n",
    "\n",
    "These assumptions work well when there is a clear margin between classes, but they may be limiting when classes are highly overlapping or not easily separable.\n",
    "\n",
    "## 3. What is the model?\n",
    "SVM is a **discriminative model** that seeks to find an optimal **hyperplane** for separating classes. For a binary classification problem in $n$-dimensional space, the hyperplane is defined as:\n",
    "$$\n",
    "w \\cdot x + b = 0\n",
    "$$\n",
    "where $w$ is the weight vector perpendicular to the hyperplane, $x$ is a feature vector, and $b$ is the bias term.\n",
    "\n",
    "The goal of SVM is to maximize the margin $M$ between the hyperplane and the nearest data points from each class, known as **support vectors**. Mathematically, this can be represented by minimizing $\\| w \\|$, subject to the constraints:\n",
    "$$\n",
    "y_i (w \\cdot x_i + b) \\geq 1\n",
    "$$\n",
    "for each training sample $(x_i, y_i)$, where $y_i \\in \\{-1, 1\\}$ indicates the class label. The hyperplane that satisfies these constraints with the maximum margin is selected as the decision boundary.\n",
    "\n",
    "In **multi-class classification**, SVM can be extended using methods such as:\n",
    "- **One-vs-One (OvO)**: Training an SVM classifier for every pair of classes.\n",
    "- **One-vs-All (OvA)**: Training an SVM classifier for each class against all other classes.\n",
    "\n",
    "The class with the highest confidence score is selected for each new instance.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "Training an SVM involves solving an **optimization problem** to find the values of $w$ and $b$ that maximize the margin while correctly classifying the data points. The optimization problem can be formulated as:\n",
    "$$\n",
    "\\min_{w, b} \\frac{1}{2} \\| w \\|^2\n",
    "$$\n",
    "subject to the constraints:\n",
    "$$\n",
    "y_i (w \\cdot x_i + b) \\geq 1 \\quad \\forall i\n",
    "$$\n",
    "\n",
    "For data that is not perfectly separable, **soft margin SVM** introduces a penalty term for misclassification by using slack variables $\\xi_i$. The optimization problem then becomes:\n",
    "$$\n",
    "\\min_{w, b} \\frac{1}{2} \\| w \\|^2 + C \\sum_{i} \\xi_i\n",
    "$$\n",
    "subject to:\n",
    "$$\n",
    "y_i (w \\cdot x_i + b) \\geq 1 - \\xi_i \\quad \\forall i\n",
    "$$\n",
    "where $C$ is a regularization parameter that balances maximizing the margin and minimizing classification errors.\n",
    "\n",
    "For **non-linear data**, the kernel trick transforms the data into a higher-dimensional space using a kernel function $K(x_i, x_j)$, such as:\n",
    "- **Linear kernel**: $K(x_i, x_j) = x_i \\cdot x_j$\n",
    "- **Polynomial kernel**: $K(x_i, x_j) = (x_i \\cdot x_j + 1)^d$\n",
    "- **Radial basis function (RBF)**: $K(x_i, x_j) = \\exp(-\\gamma \\| x_i - x_j \\|^2)$\n",
    "\n",
    "After training, SVM uses the support vectors and learned parameters to classify new inputs based on which side of the hyperplane they fall, allowing for robust classification even in complex multi-class scenarios."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 11\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;66;03m# Evaluate the SVM model\u001b[39;00m\n\u001b[1;32m     10\u001b[0m svm_train_accuracy \u001b[38;5;241m=\u001b[39m svm_model\u001b[38;5;241m.\u001b[39mscore(train_features, train_labels)\n\u001b[0;32m---> 11\u001b[0m svm_test_accuracy \u001b[38;5;241m=\u001b[39m \u001b[43msvm_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest_features\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest_labels\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSVM Train accuracy: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msvm_train_accuracy\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSVM Test accuracy: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msvm_test_accuracy\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniforge3/envs/torch/lib/python3.12/site-packages/sklearn/base.py:764\u001b[0m, in \u001b[0;36mClassifierMixin.score\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    739\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    740\u001b[0m \u001b[38;5;124;03mReturn the mean accuracy on the given test data and labels.\u001b[39;00m\n\u001b[1;32m    741\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    760\u001b[0m \u001b[38;5;124;03m    Mean accuracy of ``self.predict(X)`` w.r.t. `y`.\u001b[39;00m\n\u001b[1;32m    761\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    762\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m accuracy_score\n\u001b[0;32m--> 764\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m accuracy_score(y, \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m, sample_weight\u001b[38;5;241m=\u001b[39msample_weight)\n",
      "File \u001b[0;32m~/miniforge3/envs/torch/lib/python3.12/site-packages/sklearn/svm/_base.py:813\u001b[0m, in \u001b[0;36mBaseSVC.predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    811\u001b[0m     y \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margmax(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecision_function(X), axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m    812\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 813\u001b[0m     y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    814\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclasses_\u001b[38;5;241m.\u001b[39mtake(np\u001b[38;5;241m.\u001b[39masarray(y, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mintp))\n",
      "File \u001b[0;32m~/miniforge3/envs/torch/lib/python3.12/site-packages/sklearn/svm/_base.py:430\u001b[0m, in \u001b[0;36mBaseLibSVM.predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    428\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_for_predict(X)\n\u001b[1;32m    429\u001b[0m predict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sparse_predict \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sparse \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dense_predict\n\u001b[0;32m--> 430\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/torch/lib/python3.12/site-packages/sklearn/svm/_base.py:449\u001b[0m, in \u001b[0;36mBaseLibSVM._dense_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    441\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    442\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mX.shape[1] = \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m should be equal to \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    443\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe number of samples at training time\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    444\u001b[0m             \u001b[38;5;241m%\u001b[39m (X\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape_fit_[\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m    445\u001b[0m         )\n\u001b[1;32m    447\u001b[0m svm_type \u001b[38;5;241m=\u001b[39m LIBSVM_IMPL\u001b[38;5;241m.\u001b[39mindex(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_impl)\n\u001b[0;32m--> 449\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlibsvm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    451\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msupport_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msupport_vectors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    453\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_n_support\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    454\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dual_coef_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    455\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_intercept_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    456\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_probA\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    457\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_probB\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    458\u001b[0m \u001b[43m    \u001b[49m\u001b[43msvm_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msvm_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    459\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkernel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkernel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    460\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdegree\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdegree\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    461\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcoef0\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcoef0\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    462\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgamma\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_gamma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    463\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcache_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcache_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    464\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "# Initialize the SVM model\n",
    "svm_model = SVC()\n",
    "\n",
    "# Train the SVM model\n",
    "svm_model.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the SVM model\n",
    "svm_train_accuracy = svm_model.score(train_features, train_labels)\n",
    "svm_test_accuracy = svm_model.score(test_features, test_labels)\n",
    "\n",
    "print(f\"SVM Train accuracy: {svm_train_accuracy}\")\n",
    "print(f\"SVM Test accuracy: {svm_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest Algorithm\n",
    "\n",
    "## 1. What problem is Random Forest used to solve?\n",
    "Random Forest is a versatile supervised learning algorithm that can be used for both **classification and regression** tasks. It is particularly effective for **multi-class classification** problems where the goal is to assign one of several possible class labels to an input. Random Forest enhances the performance of individual decision trees by creating an ensemble of trees, which reduces overfitting and improves accuracy, especially when handling complex datasets.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "Random Forest assumes that:\n",
    "1. **Individual trees provide independent and complementary views** of the data, so aggregating their predictions will improve the overall accuracy and robustness.\n",
    "2. **Data can be effectively separated by multiple decision trees working together**, each capturing a different aspect of the data's structure.\n",
    "\n",
    "These assumptions allow Random Forest to handle high-dimensional and complex data well, as it averages over multiple uncorrelated trees to reduce variance and make more stable predictions.\n",
    "\n",
    "## 3. What is the model?\n",
    "Random Forest is an **ensemble model** composed of multiple decision trees, each built on a random subset of the training data. Each tree is trained independently and is slightly different due to the random selection of data and features. The model’s prediction is an **aggregate of all the individual trees’ predictions**.\n",
    "\n",
    "For **classification**, Random Forest uses **majority voting**:\n",
    "- Let each tree in the forest predict a class label for an input instance.\n",
    "- The class that receives the majority of votes across all trees is the final prediction for that instance.\n",
    "\n",
    "In **multi-class classification**, each tree provides a class prediction from among multiple possible classes, and the final predicted class is the one that appears most frequently across all trees.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "Training a Random Forest involves the following steps:\n",
    "1. **Bootstrap Sampling**: From the training dataset with $N$ samples, a **random subset with replacement** is drawn to train each individual tree. This subset is called a **bootstrap sample**. The bootstrapping process creates diversity among the trees by providing each tree with slightly different data.\n",
    "\n",
    "2. **Feature Selection at Splits**: At each split within a tree, a random subset of features (out of the total features) is considered to determine the best split. This further introduces diversity by ensuring that trees focus on different features, making them less correlated.\n",
    "\n",
    "3. **Tree Building**: Each tree is grown to its full depth without pruning, meaning that each tree will independently capture various patterns in the data. Trees are built using traditional methods like **Gini impurity** or **entropy** for classification splits, aiming to maximize the purity of nodes.\n",
    "\n",
    "4. **Aggregation**: For classification, the final output of the Random Forest is determined by **majority voting** among the trees. For each input, the class with the most votes across all trees is selected as the final class label.\n",
    "\n",
    "Overall, the Random Forest model combines the predictions of multiple uncorrelated trees, resulting in improved accuracy, stability, and resilience to overfitting, making it highly effective for multi-class classification tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Initialize the Random Forest model\n",
    "random_forest = RandomForestClassifier()\n",
    "\n",
    "# Train the Random Forest model\n",
    "random_forest.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the Random Forest model\n",
    "random_forest_train_accuracy = random_forest.score(train_features, train_labels)\n",
    "random_forest_test_accuracy = random_forest.score(test_features, test_labels)\n",
    "\n",
    "print(f\"Random Forest Train accuracy: {random_forest_train_accuracy}\")\n",
    "print(f\"Random Forest Test accuracy: {random_forest_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting Machines (GBM)\n",
    "\n",
    "## 1. What problem is Gradient Boosting Machines (GBM) used to solve?\n",
    "\n",
    "Gradient Boosting Machines (GBM) are powerful supervised learning algorithms used for both **classification and regression** tasks. They are particularly well-suited for complex **multi-class classification** problems where high predictive accuracy is required. GBMs are effective because they combine the predictions of multiple weak learners (typically shallow decision trees) to create a strong predictive model, achieving state-of-the-art results in many areas, including tabular data and structured datasets.\n",
    "\n",
    "## 2. What assumption does the model make?\n",
    "\n",
    "GBM assumes that:\n",
    "1. **Errors can be reduced iteratively**: The algorithm assumes that by sequentially building trees that focus on correcting the errors of previous trees, it can minimize overall error.\n",
    "2. **Weak learners (small trees) can be boosted**: GBM assumes that even simple models (shallow decision trees) can contribute to a highly accurate ensemble model if they are correctly trained in sequence.\n",
    "\n",
    "These assumptions enable GBM to build a model that improves over each iteration, ultimately achieving high predictive performance.\n",
    "\n",
    "## 3. What is the model?\n",
    "\n",
    "GBM is an **ensemble model** that builds a strong predictor by combining multiple weak learners, often shallow decision trees. The model works by adding trees in a sequence, where each new tree is trained to correct the errors of the previous trees. Instead of averaging or voting like in Random Forests, GBM **boosts** the model's accuracy by learning residual errors.\n",
    "\n",
    "In a **multi-class classification** context, GBM trains separate trees for each class and combines them to assign the final class with the highest probability. The final prediction is the sum of all the trees’ contributions for each class, resulting in a model that can handle multiple class labels.\n",
    "\n",
    "## 4. How is the model trained?\n",
    "\n",
    "Training a GBM model involves an **iterative, stage-wise approach** where each tree is trained to minimize the residual errors of the ensemble so far. The process is as follows:\n",
    "\n",
    "1. **Initialize the model**: The model begins by making an initial guess, often by predicting the average of the target values.\n",
    "\n",
    "2. **Calculate Residuals**: For each iteration, the residual error (the difference between the actual and predicted values) is calculated.\n",
    "\n",
    "3. **Train a Weak Learner**: A new decision tree is trained to predict the residuals from the previous model. This tree is added to the ensemble, focusing on correcting the errors made by the previous trees.\n",
    "\n",
    "4. **Update the Prediction**: The predictions are updated by adding a fraction of the new tree’s predictions to the overall model. This fraction is controlled by a parameter called the **learning rate** $\\alpha$, which regulates the contribution of each tree to prevent overfitting.\n",
    "\n",
    "5. **Repeat**: Steps 2 to 4 are repeated for a specified number of iterations or until a desired error threshold is reached. Each new tree gradually reduces the overall error, improving the model’s predictive accuracy.\n",
    "\n",
    "In mathematical terms, the model’s prediction at iteration $t$ is given by:\n",
    "$$\n",
    "F_t(x) = F_{t-1}(x) + \\alpha \\cdot h_t(x)\n",
    "$$\n",
    "where $F_t(x)$ is the updated prediction, $F_{t-1}(x)$ is the prediction from the previous iteration, $\\alpha$ is the learning rate, and $h_t(x)$ is the new tree trained on the residuals. This process continues until the model converges or reaches the maximum number of iterations.\n",
    "\n",
    "## 5. Light Gradient Boosting Machines\n",
    "\n",
    "**Light Gradient Boosting Machine (LightGBM)** is an optimized version of GBM designed for faster training on large datasets, particularly when there are many features. LightGBM introduces several innovations to improve efficiency and accuracy:\n",
    "\n",
    "- **Leaf-wise Growth**: Unlike traditional GBM that grows trees level-by-level, LightGBM grows trees leaf-by-leaf. This approach allows LightGBM to focus on the areas with the most significant errors, resulting in better accuracy and reduced computation.\n",
    "  \n",
    "- **Gradient-based One-Side Sampling (GOSS)**: LightGBM selects a subset of data with larger gradients, which helps the model focus on instances with higher errors. This reduces the amount of data that needs to be processed without sacrificing accuracy.\n",
    "  \n",
    "- **Exclusive Feature Bundling (EFB)**: LightGBM groups mutually exclusive features (those that rarely take non-zero values simultaneously) into a single feature. This feature bundling reduces the dimensionality of the data, speeding up training and reducing memory usage.\n",
    "\n",
    "In **multi-class classification**, LightGBM handles multiple classes efficiently by training separate trees for each class, similar to traditional GBM, but with the benefit of its optimized algorithms. LightGBM's improvements make it one of the fastest and most accurate gradient boosting methods available, particularly effective for large datasets with high feature dimensionality."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Initialize the Gradient Boosting model\n",
    "gbm = GradientBoostingClassifier()\n",
    "\n",
    "# Train the Gradient Boosting model\n",
    "gbm.fit(train_features, train_labels)\n",
    "\n",
    "# Evaluate the Gradient Boosting model\n",
    "gbm_train_accuracy = gbm.score(train_features, train_labels)\n",
    "gbm_test_accuracy = gbm.score(test_features, test_labels)\n",
    "\n",
    "print(f\"Gradient Boosting Train accuracy: {gbm_train_accuracy}\")\n",
    "print(f\"Gradient Boosting Test accuracy: {gbm_test_accuracy}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}