DataAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Analysis SOLOLEARN\n",
    "## Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use strings as indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    1\n",
       "b    2\n",
       "c    3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([1, 2, 3], index=['a', 'b', 'c'])  # with index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    1\n",
       "b    2\n",
       "c    3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(np.array([1, 2, 3]), index=['a', 'b', 'c'])  # with numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "series = pd.Series({'a': 1, 'b': 2, 'c': 3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "series['a']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>red_wine</th>\n",
       "      <th>white_wine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adam</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bob</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>charles</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         red_wine  white_wine\n",
       "adam            3           5\n",
       "bob             6           0\n",
       "charles         5          10"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine_dict = {\n",
    "  'red_wine': [3, 6, 5], \n",
    "  'white_wine':[5, 0, 10]\n",
    "}\n",
    "sales = pd.DataFrame(wine_dict, index=[\"adam\", \"bob\", \"charles\"])\n",
    "sales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "adam        5\n",
       "bob         0\n",
       "charles    10\n",
       "Name: white_wine, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sales['white_wine']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load sample data from Sololearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "presidents_df = pd.read_csv('https://sololearn.com/uploads/files/president_heights_party.csv',\n",
    "                            index_col='name')\n",
    "presidents_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45, 4)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "45 rows and 4 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "180"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "180 cells"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a peek at the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>George Washington</th>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>189</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>John Adams</th>\n",
       "      <td>2</td>\n",
       "      <td>61</td>\n",
       "      <td>170</td>\n",
       "      <td>federalist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thomas Jefferson</th>\n",
       "      <td>3</td>\n",
       "      <td>57</td>\n",
       "      <td>189</td>\n",
       "      <td>democratic-republican</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   order  age  height                  party\n",
       "name                                                        \n",
       "George Washington      1   57     189                   none\n",
       "John Adams             2   61     170             federalist\n",
       "Thomas Jefferson       3   57     189  democratic-republican"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>George H. W. Bush</th>\n",
       "      <td>41</td>\n",
       "      <td>64</td>\n",
       "      <td>188</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bill Clinton</th>\n",
       "      <td>42</td>\n",
       "      <td>46</td>\n",
       "      <td>188</td>\n",
       "      <td>democratic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>George W. Bush</th>\n",
       "      <td>43</td>\n",
       "      <td>54</td>\n",
       "      <td>182</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Barack Obama</th>\n",
       "      <td>44</td>\n",
       "      <td>47</td>\n",
       "      <td>185</td>\n",
       "      <td>democratic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donald J. Trump</th>\n",
       "      <td>45</td>\n",
       "      <td>70</td>\n",
       "      <td>191</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   order  age  height       party\n",
       "name                                             \n",
       "George H. W. Bush     41   64     188  republican\n",
       "Bill Clinton          42   46     188  democratic\n",
       "George W. Bush        43   54     182  republican\n",
       "Barack Obama          44   47     185  democratic\n",
       "Donald J. Trump       45   70     191  republican"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.tail()  # n defaults to be 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function `.info` column names and data types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 45 entries, George Washington to Donald J. Trump\n",
      "Data columns (total 4 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   order   45 non-null     int64 \n",
      " 1   age     45 non-null     int64 \n",
      " 2   height  45 non-null     int64 \n",
      " 3   party   45 non-null     object\n",
      "dtypes: int64(3), object(1)\n",
      "memory usage: 1.8+ KB\n"
     ]
    }
   ],
   "source": [
    "presidents_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Row selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lincoln = presidents_df.loc['Abraham Lincoln']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(lincoln)\n",
    "lincoln.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`.loc` returns a panda series with shape (n,) where n is the number of columns\n",
    "***\n",
    "slicing is also supported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Abraham Lincoln</th>\n",
       "      <td>16</td>\n",
       "      <td>52</td>\n",
       "      <td>193</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Andrew Johnson</th>\n",
       "      <td>17</td>\n",
       "      <td>56</td>\n",
       "      <td>178</td>\n",
       "      <td>national union</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ulysses S. Grant</th>\n",
       "      <td>18</td>\n",
       "      <td>46</td>\n",
       "      <td>173</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  order  age  height           party\n",
       "name                                                \n",
       "Abraham Lincoln      16   52     193      republican\n",
       "Andrew Johnson       17   56     178  national union\n",
       "Ulysses S. Grant     18   46     173      republican"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df['Abraham Lincoln':'Ulysses S. Grant']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `.iloc` to select with index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "order             16\n",
       "age               52\n",
       "height           193\n",
       "party     republican\n",
       "Name: Abraham Lincoln, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.iloc[15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Abraham Lincoln</th>\n",
       "      <td>16</td>\n",
       "      <td>52</td>\n",
       "      <td>193</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Andrew Johnson</th>\n",
       "      <td>17</td>\n",
       "      <td>56</td>\n",
       "      <td>178</td>\n",
       "      <td>national union</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ulysses S. Grant</th>\n",
       "      <td>18</td>\n",
       "      <td>46</td>\n",
       "      <td>173</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rutherford B. Hayes</th>\n",
       "      <td>19</td>\n",
       "      <td>54</td>\n",
       "      <td>174</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>James A. Garfield</th>\n",
       "      <td>20</td>\n",
       "      <td>49</td>\n",
       "      <td>183</td>\n",
       "      <td>republican</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     order  age  height           party\n",
       "name                                                   \n",
       "Abraham Lincoln         16   52     193      republican\n",
       "Andrew Johnson          17   56     178  national union\n",
       "Ulysses S. Grant        18   46     173      republican\n",
       "Rutherford B. Hayes     19   54     174      republican\n",
       "James A. Garfield       20   49     183      republican"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.iloc[15:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "above is from 16th to 20th in the data frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To fucos on column of interest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['order', 'age', 'height', 'party'], dtype='object')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "height = presidents_df['height']\n",
    "height.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>height</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>George Washington</th>\n",
       "      <td>189</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>John Adams</th>\n",
       "      <td>170</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thomas Jefferson</th>\n",
       "      <td>189</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   height  age\n",
       "name                          \n",
       "George Washington     189   57\n",
       "John Adams            170   61\n",
       "Thomas Jefferson      189   57"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df[['height', 'age']].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>George Washington</th>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>John Adams</th>\n",
       "      <td>2</td>\n",
       "      <td>61</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thomas Jefferson</th>\n",
       "      <td>3</td>\n",
       "      <td>57</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   order  age  height\n",
       "name                                 \n",
       "George Washington      1   57     189\n",
       "John Adams             2   61     170\n",
       "Thomas Jefferson       3   57     189"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.loc[:, 'order':'height'].head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### summary statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "order              1\n",
       "age               42\n",
       "height           163\n",
       "party     democratic\n",
       "dtype: object"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "order       45\n",
       "age         70\n",
       "height     193\n",
       "party     whig\n",
       "dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "order      23.022222\n",
       "age        55.000000\n",
       "height    180.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00    42.0\n",
       "0.25    51.0\n",
       "0.50    55.0\n",
       "0.75    58.0\n",
       "1.00    70.0\n",
       "Name: age, dtype: float64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df['age'].quantile(np.linspace(0, 1, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.0\n",
      "55.0\n"
     ]
    }
   ],
   "source": [
    "print(presidents_df['age'].mean())\n",
    "print(presidents_df['age'].median())  # same as presidents_df.quantile(.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "May not necessarily be the same for all datasets.\n",
    "The measure of location, median, is more robust than mean, for continuous variables as the latter is sensitive to outliers, e.g., extremely large values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43.5\n",
      "6.59545297913646\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(presidents_df['age'].var())\n",
    "print(presidents_df['age'].std())\n",
    "presidents_df['age'].var() == presidents_df['age'].std() ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "age       6.595453\n",
       "height    6.977236\n",
       "dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.loc[:,'age':].std()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`.describe` returns a table of summary statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>45.000000</td>\n",
       "      <td>45.000000</td>\n",
       "      <td>45.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>23.022222</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>180.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>13.136502</td>\n",
       "      <td>6.595453</td>\n",
       "      <td>6.977236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>163.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>12.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>175.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>23.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>182.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>34.000000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>183.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>45.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>193.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           order        age      height\n",
       "count  45.000000  45.000000   45.000000\n",
       "mean   23.022222  55.000000  180.000000\n",
       "std    13.136502   6.595453    6.977236\n",
       "min     1.000000  42.000000  163.000000\n",
       "25%    12.000000  51.000000  175.000000\n",
       "50%    23.000000  55.000000  182.000000\n",
       "75%    34.000000  58.000000  183.000000\n",
       "max    45.000000  70.000000  193.000000"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "republican               19\n",
       "democratic               15\n",
       "whig                      4\n",
       "democratic-republican     4\n",
       "federalist                1\n",
       "none                      1\n",
       "national union            1\n",
       "Name: party, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df['party'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Groupby and Aggregations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('democratic',\n",
       "                         order  age  height       party\n",
       "  name                                                 \n",
       "  Andrew Jackson             7   61     185  democratic\n",
       "  Martin Van Buren           8   54     168  democratic\n",
       "  James K. Polk             11   49     173  democratic\n",
       "  Franklin Pierce           14   48     178  democratic\n",
       "  James Buchanan            15   65     183  democratic\n",
       "  Grover Cleveland          22   47     180  democratic\n",
       "  Grover Cleveland          25   55     180  democratic\n",
       "  Woodrow Wilson            28   56     180  democratic\n",
       "  Franklin D. Roosevelt     32   51     188  democratic\n",
       "  Harry S. Truman           33   60     175  democratic\n",
       "  John F. Kennedy           35   43     183  democratic\n",
       "  Lyndon B. Johnson         36   55     193  democratic\n",
       "  Jimmy Carter              39   52     177  democratic\n",
       "  Bill Clinton              42   46     188  democratic\n",
       "  Barack Obama              44   47     185  democratic),\n",
       " ('democratic-republican',\n",
       "                     order  age  height                  party\n",
       "  name                                                        \n",
       "  Thomas Jefferson       3   57     189  democratic-republican\n",
       "  James Madison          4   57     163  democratic-republican\n",
       "  James Monroe           5   58     183  democratic-republican\n",
       "  John Quincy Adams      6   57     171  democratic-republican),\n",
       " ('federalist',\n",
       "              order  age  height       party\n",
       "  name                                      \n",
       "  John Adams      2   61     170  federalist),\n",
       " ('national union',\n",
       "                  order  age  height           party\n",
       "  name                                              \n",
       "  Andrew Johnson     17   56     178  national union),\n",
       " ('none',\n",
       "                     order  age  height party\n",
       "  name                                       \n",
       "  George Washington      1   57     189  none),\n",
       " ('republican',\n",
       "                        order  age  height       party\n",
       "  name                                                \n",
       "  Abraham Lincoln          16   52     193  republican\n",
       "  Ulysses S. Grant         18   46     173  republican\n",
       "  Rutherford B. Hayes      19   54     174  republican\n",
       "  James A. Garfield        20   49     183  republican\n",
       "  Chester A. Arthur        21   51     183  republican\n",
       "  Benjamin Harrison        23   55     168  republican\n",
       "  William McKinley         25   54     170  republican\n",
       "  Theodore Roosevelt       26   42     178  republican\n",
       "  William Howard Taft      27   51     182  republican\n",
       "  Warren G. Harding        29   55     183  republican\n",
       "  Calvin Coolidge          30   51     178  republican\n",
       "  Herbert Hoover           31   54     182  republican\n",
       "  Dwight D. Eisenhower     34   62     179  republican\n",
       "  Richard Nixon            37   56     182  republican\n",
       "  Gerald Ford              38   61     183  republican\n",
       "  Ronald Reagan            40   69     185  republican\n",
       "  George H. W. Bush        41   64     188  republican\n",
       "  George W. Bush           43   54     182  republican\n",
       "  Donald J. Trump          45   70     191  republican),\n",
       " ('whig',\n",
       "                          order  age  height party\n",
       "  name                                            \n",
       "  William Henry Harrison      9   68     173  whig\n",
       "  John Tyler                 10   51     183  whig\n",
       "  Zachary Taylor             12   64     173  whig\n",
       "  Millard Fillmore           13   50     175  whig)]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(presidents_df.groupby('party'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also perform multiple operations on the groupby object (and dataframe) using `.agg()` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>age</th>\n",
       "      <th>height</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>party</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>democratic</th>\n",
       "      <td>26.066667</td>\n",
       "      <td>52.600000</td>\n",
       "      <td>181.066667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>democratic-republican</th>\n",
       "      <td>4.500000</td>\n",
       "      <td>57.250000</td>\n",
       "      <td>176.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>federalist</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>170.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>national union</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>178.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>189.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>republican</th>\n",
       "      <td>29.631579</td>\n",
       "      <td>55.263158</td>\n",
       "      <td>180.894737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whig</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>58.250000</td>\n",
       "      <td>176.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           order        age      height\n",
       "party                                                  \n",
       "democratic             26.066667  52.600000  181.066667\n",
       "democratic-republican   4.500000  57.250000  176.500000\n",
       "federalist              2.000000  61.000000  170.000000\n",
       "national union         17.000000  56.000000  178.000000\n",
       "none                    1.000000  57.000000  189.000000\n",
       "republican             29.631579  55.263158  180.894737\n",
       "whig                   11.000000  58.250000  176.000000"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.groupby('party').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>median</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>party</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>democratic</th>\n",
       "      <td>168</td>\n",
       "      <td>180</td>\n",
       "      <td>193</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>democratic-republican</th>\n",
       "      <td>163</td>\n",
       "      <td>177</td>\n",
       "      <td>189</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>federalist</th>\n",
       "      <td>170</td>\n",
       "      <td>170</td>\n",
       "      <td>170</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>national union</th>\n",
       "      <td>178</td>\n",
       "      <td>178</td>\n",
       "      <td>178</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>189</td>\n",
       "      <td>189</td>\n",
       "      <td>189</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>republican</th>\n",
       "      <td>168</td>\n",
       "      <td>182</td>\n",
       "      <td>193</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whig</th>\n",
       "      <td>173</td>\n",
       "      <td>174</td>\n",
       "      <td>183</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       min  median  max  count\n",
       "party                                         \n",
       "democratic             168     180  193     15\n",
       "democratic-republican  163     177  189      4\n",
       "federalist             170     170  170      1\n",
       "national union         178     178  178      1\n",
       "none                   189     189  189      1\n",
       "republican             168     182  193     19\n",
       "whig                   173     174  183      4"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.groupby('party')['height'].agg(['min', np.median, max])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th colspan=\"2\" halign=\"left\">height</th>\n",
       "      <th colspan=\"2\" halign=\"left\">age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>party</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>democratic</th>\n",
       "      <td>15</td>\n",
       "      <td>180</td>\n",
       "      <td>181.066667</td>\n",
       "      <td>43</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>democratic-republican</th>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "      <td>176.500000</td>\n",
       "      <td>57</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>federalist</th>\n",
       "      <td>1</td>\n",
       "      <td>170</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>61</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>national union</th>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>178.000000</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "      <td>189.000000</td>\n",
       "      <td>57</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>republican</th>\n",
       "      <td>19</td>\n",
       "      <td>182</td>\n",
       "      <td>180.894737</td>\n",
       "      <td>42</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whig</th>\n",
       "      <td>4</td>\n",
       "      <td>174</td>\n",
       "      <td>176.000000</td>\n",
       "      <td>50</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      order height             age    \n",
       "                      count median        mean min max\n",
       "party                                                 \n",
       "democratic               15    180  181.066667  43  65\n",
       "democratic-republican     4    177  176.500000  57  58\n",
       "federalist                1    170  170.000000  61  61\n",
       "national union            1    178  178.000000  56  56\n",
       "none                      1    189  189.000000  57  57\n",
       "republican               19    182  180.894737  42  70\n",
       "whig                      4    174  176.000000  50  68"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presidents_df.groupby('party').agg({'order': 'count',\n",
    "                                    'height': [np.median, np.mean],\n",
    "                                    'age': [min, max]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}