process_afdb_2017_vfdb_cudb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR  \n",
       "0      57.471264  \n",
       "1      57.471264  \n",
       "2      57.251908  \n",
       "3      57.251908  \n",
       "4      57.034221  \n",
       "...          ...  \n",
       "3723  164.835165  \n",
       "3724   43.731778  \n",
       "3725   43.859649  \n",
       "3726   43.604651  \n",
       "3727   43.352601  \n",
       "\n",
       "[3728 rows x 8 columns]"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv\", index_col = 0)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 16s, sys: 3.41 s, total: 1min 20s\n",
      "Wall time: 1min 21s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>14990</th>\n",
       "      <th>14991</th>\n",
       "      <th>14992</th>\n",
       "      <th>14993</th>\n",
       "      <th>14994</th>\n",
       "      <th>14995</th>\n",
       "      <th>14996</th>\n",
       "      <th>14997</th>\n",
       "      <th>14998</th>\n",
       "      <th>14999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14040</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14041 rows × 15001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       af  0  1  2  3  4  5  6  7  8  ...  14990  14991  14992  14993  14994  \\\n",
       "0       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "1       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "2       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "3       0  0  0  0  0  1  0  0  0  0  ...      0      0      0      0      0   \n",
       "4       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "...    .. .. .. .. .. .. .. .. .. ..  ...    ...    ...    ...    ...    ...   \n",
       "14036   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14037   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14038   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14039   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14040   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "\n",
       "       14995  14996  14997  14998  14999  \n",
       "0          0      0      0      0      0  \n",
       "1          0      0      0      0      0  \n",
       "2          0      0      0      0      0  \n",
       "3          0      0      0      0      0  \n",
       "4          0      0      0      1      0  \n",
       "...      ...    ...    ...    ...    ...  \n",
       "14036      0      0      0      0      0  \n",
       "14037      0      0      0      0      0  \n",
       "14038      0      0      0      0      0  \n",
       "14039      0      0      0      0      0  \n",
       "14040      0      0      0      0      0  \n",
       "\n",
       "[14041 rows x 15001 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "final = pd.read_csv(\"/hdd/physio/af/rpeakdf.csv\", index_col = 0)\n",
    "final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.isfinite(df).values.all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "df = df.dropna(how=\"all\", inplace = False)\n",
    "df.drop(columns = ['mean_hr', 'max_hr', 'triangular_index'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>mean_nni</th>\n",
       "      <th>sdnn</th>\n",
       "      <th>sdsd</th>\n",
       "      <th>nni_50</th>\n",
       "      <th>nni_20</th>\n",
       "      <th>rmssd</th>\n",
       "      <th>median_nni</th>\n",
       "      <th>range_nni</th>\n",
       "      <th>cvsd</th>\n",
       "      <th>cvnni</th>\n",
       "      <th>min_hr</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>748.250000</td>\n",
       "      <td>349.501907</td>\n",
       "      <td>6.649979</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>270.415113</td>\n",
       "      <td>752.0</td>\n",
       "      <td>811.0</td>\n",
       "      <td>0.361397</td>\n",
       "      <td>0.467092</td>\n",
       "      <td>52.173913</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>656.250000</td>\n",
       "      <td>338.112777</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>262.003817</td>\n",
       "      <td>655.5</td>\n",
       "      <td>786.0</td>\n",
       "      <td>0.399244</td>\n",
       "      <td>0.515219</td>\n",
       "      <td>57.142857</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>690.400000</td>\n",
       "      <td>419.965237</td>\n",
       "      <td>5.117372</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>266.299174</td>\n",
       "      <td>689.0</td>\n",
       "      <td>1065.0</td>\n",
       "      <td>0.385717</td>\n",
       "      <td>0.608293</td>\n",
       "      <td>48.939641</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>597.600000</td>\n",
       "      <td>422.650328</td>\n",
       "      <td>5.123475</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>266.549245</td>\n",
       "      <td>601.0</td>\n",
       "      <td>1066.0</td>\n",
       "      <td>0.446033</td>\n",
       "      <td>0.707246</td>\n",
       "      <td>53.097345</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>640.750000</td>\n",
       "      <td>345.356217</td>\n",
       "      <td>4.642796</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>267.706929</td>\n",
       "      <td>638.0</td>\n",
       "      <td>803.0</td>\n",
       "      <td>0.417802</td>\n",
       "      <td>0.538987</td>\n",
       "      <td>57.416268</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>459.400000</td>\n",
       "      <td>314.091229</td>\n",
       "      <td>102.611829</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>221.900315</td>\n",
       "      <td>351.0</td>\n",
       "      <td>787.0</td>\n",
       "      <td>0.483022</td>\n",
       "      <td>0.683699</td>\n",
       "      <td>63.965885</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>657.666667</td>\n",
       "      <td>345.503015</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>345.509045</td>\n",
       "      <td>656.0</td>\n",
       "      <td>691.0</td>\n",
       "      <td>0.525356</td>\n",
       "      <td>0.525347</td>\n",
       "      <td>59.760956</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>549.000000</td>\n",
       "      <td>444.621187</td>\n",
       "      <td>2.494438</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>344.675693</td>\n",
       "      <td>550.0</td>\n",
       "      <td>1034.0</td>\n",
       "      <td>0.627825</td>\n",
       "      <td>0.809875</td>\n",
       "      <td>56.338028</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>612.500000</td>\n",
       "      <td>444.877137</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>344.667956</td>\n",
       "      <td>612.0</td>\n",
       "      <td>1034.0</td>\n",
       "      <td>0.562723</td>\n",
       "      <td>0.726330</td>\n",
       "      <td>53.097345</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>680.250000</td>\n",
       "      <td>449.655702</td>\n",
       "      <td>2.054805</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>348.339394</td>\n",
       "      <td>679.0</td>\n",
       "      <td>1045.0</td>\n",
       "      <td>0.512076</td>\n",
       "      <td>0.661015</td>\n",
       "      <td>49.833887</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf    mean_nni        sdnn        sdsd  nni_50  nni_20       rmssd  \\\n",
       "0      0  748.250000  349.501907    6.649979     3.0     3.0  270.415113   \n",
       "1      0  656.250000  338.112777    1.414214     3.0     3.0  262.003817   \n",
       "2      0  690.400000  419.965237    5.117372     4.0     4.0  266.299174   \n",
       "3      0  597.600000  422.650328    5.123475     4.0     4.0  266.549245   \n",
       "4      0  640.750000  345.356217    4.642796     3.0     3.0  267.706929   \n",
       "...   ..         ...         ...         ...     ...     ...         ...   \n",
       "3723   1  459.400000  314.091229  102.611829     4.0     4.0  221.900315   \n",
       "3724   1  657.666667  345.503015    2.500000     2.0     2.0  345.509045   \n",
       "3725   1  549.000000  444.621187    2.494438     3.0     3.0  344.675693   \n",
       "3726   1  612.500000  444.877137    0.942809     3.0     3.0  344.667956   \n",
       "3727   1  680.250000  449.655702    2.054805     3.0     3.0  348.339394   \n",
       "\n",
       "      median_nni  range_nni      cvsd     cvnni     min_hr           lf  \\\n",
       "0          752.0      811.0  0.361397  0.467092  52.173913  1939.478779   \n",
       "1          655.5      786.0  0.399244  0.515219  57.142857  1712.922367   \n",
       "2          689.0     1065.0  0.385717  0.608293  48.939641  4196.568751   \n",
       "3          601.0     1066.0  0.446033  0.707246  53.097345  3692.942083   \n",
       "4          638.0      803.0  0.417802  0.538987  57.416268  1816.833024   \n",
       "...          ...        ...       ...       ...        ...          ...   \n",
       "3723       351.0      787.0  0.483022  0.683699  63.965885  1056.532665   \n",
       "3724       656.0      691.0  0.525356  0.525347  59.760956  1200.645300   \n",
       "3725       550.0     1034.0  0.627825  0.809875  56.338028  3101.967972   \n",
       "3726       612.0     1034.0  0.562723  0.726330  53.097345  3197.495564   \n",
       "3727       679.0     1045.0  0.512076  0.661015  49.833887  3363.977055   \n",
       "\n",
       "                hf   total_power         vlf  \n",
       "0      8359.852738  10740.573054  441.241538  \n",
       "1      7112.985993   9249.068433  423.160073  \n",
       "2     15537.499334  20534.333244  800.265159  \n",
       "3     13527.269270  18097.231413  877.020061  \n",
       "4      7521.108213   9787.482645  449.541409  \n",
       "...            ...           ...         ...  \n",
       "3723   6478.046782   7743.907815  209.328367  \n",
       "3724   3961.072822   5514.092053  352.373932  \n",
       "3725  10454.393243  14430.419775  874.058561  \n",
       "3726  11879.347598  15915.072325  838.229163  \n",
       "3727  13430.154948  17599.920092  805.788089  \n",
       "\n",
       "[3728 rows x 16 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop(columns = ['pnni_50', 'pnni_20'], inplace = True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>mean_nni</th>\n",
       "      <th>sdnn</th>\n",
       "      <th>sdsd</th>\n",
       "      <th>nni_50</th>\n",
       "      <th>nni_20</th>\n",
       "      <th>rmssd</th>\n",
       "      <th>median_nni</th>\n",
       "      <th>range_nni</th>\n",
       "      <th>cvsd</th>\n",
       "      <th>cvnni</th>\n",
       "      <th>min_hr</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>748.250000</td>\n",
       "      <td>349.501907</td>\n",
       "      <td>6.649979</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>270.415113</td>\n",
       "      <td>752.0</td>\n",
       "      <td>811.0</td>\n",
       "      <td>0.361397</td>\n",
       "      <td>0.467092</td>\n",
       "      <td>52.173913</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>656.250000</td>\n",
       "      <td>338.112777</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>262.003817</td>\n",
       "      <td>655.5</td>\n",
       "      <td>786.0</td>\n",
       "      <td>0.399244</td>\n",
       "      <td>0.515219</td>\n",
       "      <td>57.142857</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>690.400000</td>\n",
       "      <td>419.965237</td>\n",
       "      <td>5.117372</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>266.299174</td>\n",
       "      <td>689.0</td>\n",
       "      <td>1065.0</td>\n",
       "      <td>0.385717</td>\n",
       "      <td>0.608293</td>\n",
       "      <td>48.939641</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>597.600000</td>\n",
       "      <td>422.650328</td>\n",
       "      <td>5.123475</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>266.549245</td>\n",
       "      <td>601.0</td>\n",
       "      <td>1066.0</td>\n",
       "      <td>0.446033</td>\n",
       "      <td>0.707246</td>\n",
       "      <td>53.097345</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>640.750000</td>\n",
       "      <td>345.356217</td>\n",
       "      <td>4.642796</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>267.706929</td>\n",
       "      <td>638.0</td>\n",
       "      <td>803.0</td>\n",
       "      <td>0.417802</td>\n",
       "      <td>0.538987</td>\n",
       "      <td>57.416268</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>459.400000</td>\n",
       "      <td>314.091229</td>\n",
       "      <td>102.611829</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>221.900315</td>\n",
       "      <td>351.0</td>\n",
       "      <td>787.0</td>\n",
       "      <td>0.483022</td>\n",
       "      <td>0.683699</td>\n",
       "      <td>63.965885</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>657.666667</td>\n",
       "      <td>345.503015</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>345.509045</td>\n",
       "      <td>656.0</td>\n",
       "      <td>691.0</td>\n",
       "      <td>0.525356</td>\n",
       "      <td>0.525347</td>\n",
       "      <td>59.760956</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>549.000000</td>\n",
       "      <td>444.621187</td>\n",
       "      <td>2.494438</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>344.675693</td>\n",
       "      <td>550.0</td>\n",
       "      <td>1034.0</td>\n",
       "      <td>0.627825</td>\n",
       "      <td>0.809875</td>\n",
       "      <td>56.338028</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>612.500000</td>\n",
       "      <td>444.877137</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>344.667956</td>\n",
       "      <td>612.0</td>\n",
       "      <td>1034.0</td>\n",
       "      <td>0.562723</td>\n",
       "      <td>0.726330</td>\n",
       "      <td>53.097345</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>680.250000</td>\n",
       "      <td>449.655702</td>\n",
       "      <td>2.054805</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>348.339394</td>\n",
       "      <td>679.0</td>\n",
       "      <td>1045.0</td>\n",
       "      <td>0.512076</td>\n",
       "      <td>0.661015</td>\n",
       "      <td>49.833887</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf    mean_nni        sdnn        sdsd  nni_50  nni_20       rmssd  \\\n",
       "0      0  748.250000  349.501907    6.649979     3.0     3.0  270.415113   \n",
       "1      0  656.250000  338.112777    1.414214     3.0     3.0  262.003817   \n",
       "2      0  690.400000  419.965237    5.117372     4.0     4.0  266.299174   \n",
       "3      0  597.600000  422.650328    5.123475     4.0     4.0  266.549245   \n",
       "4      0  640.750000  345.356217    4.642796     3.0     3.0  267.706929   \n",
       "...   ..         ...         ...         ...     ...     ...         ...   \n",
       "3723   1  459.400000  314.091229  102.611829     4.0     4.0  221.900315   \n",
       "3724   1  657.666667  345.503015    2.500000     2.0     2.0  345.509045   \n",
       "3725   1  549.000000  444.621187    2.494438     3.0     3.0  344.675693   \n",
       "3726   1  612.500000  444.877137    0.942809     3.0     3.0  344.667956   \n",
       "3727   1  680.250000  449.655702    2.054805     3.0     3.0  348.339394   \n",
       "\n",
       "      median_nni  range_nni      cvsd     cvnni     min_hr           lf  \\\n",
       "0          752.0      811.0  0.361397  0.467092  52.173913  1939.478779   \n",
       "1          655.5      786.0  0.399244  0.515219  57.142857  1712.922367   \n",
       "2          689.0     1065.0  0.385717  0.608293  48.939641  4196.568751   \n",
       "3          601.0     1066.0  0.446033  0.707246  53.097345  3692.942083   \n",
       "4          638.0      803.0  0.417802  0.538987  57.416268  1816.833024   \n",
       "...          ...        ...       ...       ...        ...          ...   \n",
       "3723       351.0      787.0  0.483022  0.683699  63.965885  1056.532665   \n",
       "3724       656.0      691.0  0.525356  0.525347  59.760956  1200.645300   \n",
       "3725       550.0     1034.0  0.627825  0.809875  56.338028  3101.967972   \n",
       "3726       612.0     1034.0  0.562723  0.726330  53.097345  3197.495564   \n",
       "3727       679.0     1045.0  0.512076  0.661015  49.833887  3363.977055   \n",
       "\n",
       "                hf   total_power         vlf  \n",
       "0      8359.852738  10740.573054  441.241538  \n",
       "1      7112.985993   9249.068433  423.160073  \n",
       "2     15537.499334  20534.333244  800.265159  \n",
       "3     13527.269270  18097.231413  877.020061  \n",
       "4      7521.108213   9787.482645  449.541409  \n",
       "...            ...           ...         ...  \n",
       "3723   6478.046782   7743.907815  209.328367  \n",
       "3724   3961.072822   5514.092053  352.373932  \n",
       "3725  10454.393243  14430.419775  874.058561  \n",
       "3726  11879.347598  15915.072325  838.229163  \n",
       "3727  13430.154948  17599.920092  805.788089  \n",
       "\n",
       "[3728 rows x 16 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df = df[df.columns[:17]]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into train and test set\n",
    "train, test = train_test_split(finaldf, test_size=0.2, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.to_numpy()\n",
    "# test2 = test2.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try8\"\n",
    "# path = \"/hdd/physio/edgeml/examples/pytorch/FastCells/tryforbonsai1\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "# np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000000e+00, 5.85500000e+02, 3.83496632e+02, ...,\n",
       "        2.96532741e+02, 5.91500000e+02, 8.89000000e+02],\n",
       "       [0.00000000e+00, 6.07571429e+02, 3.93952559e+02, ...,\n",
       "        1.82528080e+02, 6.03000000e+02, 1.09500000e+03],\n",
       "       [0.00000000e+00, 5.88000000e+02, 3.63692570e+02, ...,\n",
       "        1.48433246e+02, 5.87000000e+02, 1.03900000e+03],\n",
       "       ...,\n",
       "       [1.00000000e+00, 6.39846154e+02, 3.60199215e+02, ...,\n",
       "        9.24504372e+01, 6.42000000e+02, 1.10900000e+03],\n",
       "       [0.00000000e+00, 6.22833333e+02, 3.43009281e+02, ...,\n",
       "        1.83412650e+02, 6.25000000e+02, 9.17000000e+02],\n",
       "       [0.00000000e+00, 6.28125000e+02, 3.65411485e+02, ...,\n",
       "        1.49143268e+02, 6.28500000e+02, 1.04400000e+03]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><ins>NORMALIZATION:</ins></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/hdd/physio/vf/finaldfs/trybonsaifeatures.csv\", index_col = 0)\n",
    "df.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "df = df.dropna(how=\"any\", inplace = False)\n",
    "# df.drop(columns = ['mean_hr', 'max_hr', 'triangular_index', 'pnni_50', 'pnni_20'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR  \n",
       "0      57.471264  \n",
       "1      57.471264  \n",
       "2      57.251908  \n",
       "3      57.251908  \n",
       "4      57.034221  \n",
       "...          ...  \n",
       "3723  164.835165  \n",
       "3724   43.731778  \n",
       "3725   43.859649  \n",
       "3726   43.604651  \n",
       "3727   43.352601  \n",
       "\n",
       "[3728 rows x 8 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(finaldf, test_size=0.2, random_state=42, shuffle=True)\n",
    "# scaler = MinMaxScaler()\n",
    "train[train.columns[1:]] = scaler.fit_transform(train[train.columns[1:]])\n",
    "test[test.columns[1:]] = scaler.transform(test[test.columns[1:]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.to_numpy()\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try1-af\"\n",
    "# path = \"/hdd/physio/edgeml/examples/pytorch/FastCells/tryforbonsai1\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 46 s, sys: 2.91 s, total: 48.9 s\n",
      "Wall time: 48.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def featureExtraction(df, fs):\n",
    "    df1 = df.af.copy()\n",
    "    df.drop(columns = ['af'], inplace = True)\n",
    "    final = list()\n",
    "    for i in range(len(df)):\n",
    "        peaks = []\n",
    "        peaks = df.loc[i][df.loc[i] == 1].index.values\n",
    "        peaks = [int(x) for x in peaks]\n",
    "        if (len(peaks) == 0 or len(peaks) == 1):\n",
    "            continue\n",
    "        af = df1.loc[i]\n",
    "        rr = np.diff(peaks) / fs \n",
    "        hr = 60 / rr\n",
    "        rmssd = np.sqrt(np.mean(np.square(rr)))\n",
    "        sdnn = np.std(rr)\n",
    "        mean_rr = np.mean(rr)\n",
    "        mean_hr = np.mean(hr)\n",
    "        std_hr = np.std(hr)\n",
    "        min_hr = np.min(hr)\n",
    "        max_hr = np.max(hr)\n",
    "#         print(rmssd, sdnn, mean_rr, mean_hr, std_hr, min_hr, max_hr); break\n",
    "        single_window_df = pd.DataFrame([[rmssd, sdnn, mean_rr, mean_hr, std_hr, min_hr, max_hr]], columns = ['RMSSD', 'STDNN', 'MEAN_RR', 'MEAN_HR', 'STD_HR', 'MIN_HR', 'MAX_HR'])\n",
    "#         single_window_df = single_window_df.transpose()\n",
    "#         columns = single_window_df.iloc[0]\n",
    "#         single_window_df = single_window_df[1:]\n",
    "#         single_window_df.columns = columns\n",
    "#         single_window_df.drop(columns = ['std_hr', 'tinn', 'lf_hf_ratio', 'lfnu', 'hfnu'], inplace = True)\n",
    "        single_window_df.insert(0, 'vf', af)\n",
    "        final.append(single_window_df)\n",
    "        del single_window_df\n",
    "    return pd.concat(final)\n",
    "\n",
    "# start = time()\n",
    "finaldf = featureExtraction(final, 250)\n",
    "finaldf.reset_index(drop = True, inplace = True)\n",
    "# print(\"Executed in {0} seconds\".format((time() - start)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.720403</td>\n",
       "      <td>0.199647</td>\n",
       "      <td>0.692186</td>\n",
       "      <td>95.439177</td>\n",
       "      <td>31.962709</td>\n",
       "      <td>52.264808</td>\n",
       "      <td>202.702703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.685475</td>\n",
       "      <td>0.209389</td>\n",
       "      <td>0.652711</td>\n",
       "      <td>101.701272</td>\n",
       "      <td>33.196907</td>\n",
       "      <td>50.675676</td>\n",
       "      <td>254.237288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.718068</td>\n",
       "      <td>0.221976</td>\n",
       "      <td>0.682897</td>\n",
       "      <td>96.715793</td>\n",
       "      <td>28.657022</td>\n",
       "      <td>49.342105</td>\n",
       "      <td>170.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.624761</td>\n",
       "      <td>0.193625</td>\n",
       "      <td>0.594000</td>\n",
       "      <td>112.472761</td>\n",
       "      <td>36.208551</td>\n",
       "      <td>60.728745</td>\n",
       "      <td>194.805195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.730429</td>\n",
       "      <td>0.230748</td>\n",
       "      <td>0.693023</td>\n",
       "      <td>95.938086</td>\n",
       "      <td>29.934269</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>192.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13736</th>\n",
       "      <td>1</td>\n",
       "      <td>0.590872</td>\n",
       "      <td>0.144073</td>\n",
       "      <td>0.573038</td>\n",
       "      <td>111.217703</td>\n",
       "      <td>28.102676</td>\n",
       "      <td>62.500000</td>\n",
       "      <td>267.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13737</th>\n",
       "      <td>1</td>\n",
       "      <td>0.581287</td>\n",
       "      <td>0.174410</td>\n",
       "      <td>0.554505</td>\n",
       "      <td>122.422290</td>\n",
       "      <td>50.413662</td>\n",
       "      <td>61.475410</td>\n",
       "      <td>300.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13738</th>\n",
       "      <td>1</td>\n",
       "      <td>0.608186</td>\n",
       "      <td>0.160637</td>\n",
       "      <td>0.586588</td>\n",
       "      <td>111.637412</td>\n",
       "      <td>38.230319</td>\n",
       "      <td>55.762082</td>\n",
       "      <td>300.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13739</th>\n",
       "      <td>1</td>\n",
       "      <td>0.596773</td>\n",
       "      <td>0.155941</td>\n",
       "      <td>0.576039</td>\n",
       "      <td>112.426787</td>\n",
       "      <td>33.248685</td>\n",
       "      <td>63.025210</td>\n",
       "      <td>245.901639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13740</th>\n",
       "      <td>1</td>\n",
       "      <td>0.577364</td>\n",
       "      <td>0.348516</td>\n",
       "      <td>0.460310</td>\n",
       "      <td>164.467461</td>\n",
       "      <td>66.663040</td>\n",
       "      <td>17.026107</td>\n",
       "      <td>300.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13741 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       vf     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0       0  0.720403  0.199647  0.692186   95.439177  31.962709  52.264808   \n",
       "1       0  0.685475  0.209389  0.652711  101.701272  33.196907  50.675676   \n",
       "2       0  0.718068  0.221976  0.682897   96.715793  28.657022  49.342105   \n",
       "3       0  0.624761  0.193625  0.594000  112.472761  36.208551  60.728745   \n",
       "4       0  0.730429  0.230748  0.693023   95.938086  29.934269  43.859649   \n",
       "...    ..       ...       ...       ...         ...        ...        ...   \n",
       "13736   1  0.590872  0.144073  0.573038  111.217703  28.102676  62.500000   \n",
       "13737   1  0.581287  0.174410  0.554505  122.422290  50.413662  61.475410   \n",
       "13738   1  0.608186  0.160637  0.586588  111.637412  38.230319  55.762082   \n",
       "13739   1  0.596773  0.155941  0.576039  112.426787  33.248685  63.025210   \n",
       "13740   1  0.577364  0.348516  0.460310  164.467461  66.663040  17.026107   \n",
       "\n",
       "           MAX_HR  \n",
       "0      202.702703  \n",
       "1      254.237288  \n",
       "2      170.454545  \n",
       "3      194.805195  \n",
       "4      192.307692  \n",
       "...           ...  \n",
       "13736  267.857143  \n",
       "13737  300.000000  \n",
       "13738  300.000000  \n",
       "13739  245.901639  \n",
       "13740  300.000000  \n",
       "\n",
       "[13741 rows x 8 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finaldf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>14990</th>\n",
       "      <th>14991</th>\n",
       "      <th>14992</th>\n",
       "      <th>14993</th>\n",
       "      <th>14994</th>\n",
       "      <th>14995</th>\n",
       "      <th>14996</th>\n",
       "      <th>14997</th>\n",
       "      <th>14998</th>\n",
       "      <th>14999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14040</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14041 rows × 15001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       af  0  1  2  3  4  5  6  7  8  ...  14990  14991  14992  14993  14994  \\\n",
       "0       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "1       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "2       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "3       0  0  0  0  0  1  0  0  0  0  ...      0      0      0      0      0   \n",
       "4       0  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "...    .. .. .. .. .. .. .. .. .. ..  ...    ...    ...    ...    ...    ...   \n",
       "14036   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14037   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14038   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14039   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "14040   1  0  0  0  0  0  0  0  0  0  ...      0      0      0      0      0   \n",
       "\n",
       "       14995  14996  14997  14998  14999  \n",
       "0          0      0      0      0      0  \n",
       "1          0      0      0      0      0  \n",
       "2          0      0      0      0      0  \n",
       "3          0      0      0      0      0  \n",
       "4          0      0      0      1      0  \n",
       "...      ...    ...    ...    ...    ...  \n",
       "14036      0      0      0      0      0  \n",
       "14037      0      0      0      0      0  \n",
       "14038      0      0      0      0      0  \n",
       "14039      0      0      0      0      0  \n",
       "14040      0      0      0      0      0  \n",
       "\n",
       "[14041 rows x 15001 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/hdd/physio/af/rpeakdf.csv\", index_col = 0)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.45 s, sys: 727 ms, total: 5.17 s\n",
      "Wall time: 5.37 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>1490</th>\n",
       "      <th>1491</th>\n",
       "      <th>1492</th>\n",
       "      <th>1493</th>\n",
       "      <th>1494</th>\n",
       "      <th>1495</th>\n",
       "      <th>1496</th>\n",
       "      <th>1497</th>\n",
       "      <th>1498</th>\n",
       "      <th>1499</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.298182</td>\n",
       "      <td>-0.288636</td>\n",
       "      <td>-0.152727</td>\n",
       "      <td>-0.203182</td>\n",
       "      <td>-0.275909</td>\n",
       "      <td>-0.091818</td>\n",
       "      <td>-0.853182</td>\n",
       "      <td>-0.425909</td>\n",
       "      <td>-0.460909</td>\n",
       "      <td>...</td>\n",
       "      <td>0.398636</td>\n",
       "      <td>0.518182</td>\n",
       "      <td>0.453182</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.640909</td>\n",
       "      <td>-0.179545</td>\n",
       "      <td>0.252727</td>\n",
       "      <td>0.228636</td>\n",
       "      <td>0.143182</td>\n",
       "      <td>-0.0355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.308182</td>\n",
       "      <td>-0.372727</td>\n",
       "      <td>0.012727</td>\n",
       "      <td>0.195909</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.100455</td>\n",
       "      <td>0.054091</td>\n",
       "      <td>0.027727</td>\n",
       "      <td>0.025909</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.787727</td>\n",
       "      <td>-0.485000</td>\n",
       "      <td>-0.438182</td>\n",
       "      <td>-0.430909</td>\n",
       "      <td>-0.452273</td>\n",
       "      <td>-0.454545</td>\n",
       "      <td>-0.445909</td>\n",
       "      <td>-0.428182</td>\n",
       "      <td>-0.414545</td>\n",
       "      <td>-0.3965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.240909</td>\n",
       "      <td>-0.326818</td>\n",
       "      <td>-0.219091</td>\n",
       "      <td>-0.444091</td>\n",
       "      <td>-0.686818</td>\n",
       "      <td>-0.429091</td>\n",
       "      <td>-0.436818</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.717273</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.124091</td>\n",
       "      <td>-0.152273</td>\n",
       "      <td>-0.293636</td>\n",
       "      <td>-0.414091</td>\n",
       "      <td>-0.443182</td>\n",
       "      <td>-0.370455</td>\n",
       "      <td>-0.036818</td>\n",
       "      <td>-0.098636</td>\n",
       "      <td>-0.185000</td>\n",
       "      <td>0.1520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.935000</td>\n",
       "      <td>-0.363636</td>\n",
       "      <td>-0.209091</td>\n",
       "      <td>-0.270000</td>\n",
       "      <td>-0.517727</td>\n",
       "      <td>-0.579091</td>\n",
       "      <td>-0.535909</td>\n",
       "      <td>-0.272727</td>\n",
       "      <td>-0.157727</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.634091</td>\n",
       "      <td>-0.386364</td>\n",
       "      <td>-0.332273</td>\n",
       "      <td>-0.392273</td>\n",
       "      <td>-0.492727</td>\n",
       "      <td>-0.485455</td>\n",
       "      <td>-0.477727</td>\n",
       "      <td>-0.489545</td>\n",
       "      <td>-0.484091</td>\n",
       "      <td>-0.4805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.493636</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.511818</td>\n",
       "      <td>-0.500455</td>\n",
       "      <td>-0.495455</td>\n",
       "      <td>-0.451818</td>\n",
       "      <td>-0.267727</td>\n",
       "      <td>-0.538636</td>\n",
       "      <td>-0.310909</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.727727</td>\n",
       "      <td>-0.699545</td>\n",
       "      <td>-0.652273</td>\n",
       "      <td>-0.595000</td>\n",
       "      <td>-0.542273</td>\n",
       "      <td>-0.493636</td>\n",
       "      <td>-0.468636</td>\n",
       "      <td>-0.451364</td>\n",
       "      <td>-0.437727</td>\n",
       "      <td>-0.4220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.266818</td>\n",
       "      <td>-0.125000</td>\n",
       "      <td>-0.283636</td>\n",
       "      <td>-0.163182</td>\n",
       "      <td>-0.210000</td>\n",
       "      <td>-0.207273</td>\n",
       "      <td>-0.259545</td>\n",
       "      <td>-0.276364</td>\n",
       "      <td>-0.111364</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.385909</td>\n",
       "      <td>-0.204545</td>\n",
       "      <td>-0.520909</td>\n",
       "      <td>-0.342727</td>\n",
       "      <td>-0.254545</td>\n",
       "      <td>-0.534091</td>\n",
       "      <td>-0.207727</td>\n",
       "      <td>-0.458182</td>\n",
       "      <td>-0.441818</td>\n",
       "      <td>-0.2315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.526818</td>\n",
       "      <td>-0.296364</td>\n",
       "      <td>-0.358182</td>\n",
       "      <td>-0.539091</td>\n",
       "      <td>-0.261818</td>\n",
       "      <td>-0.445455</td>\n",
       "      <td>-0.464091</td>\n",
       "      <td>-0.241818</td>\n",
       "      <td>-0.518636</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.213636</td>\n",
       "      <td>-0.496818</td>\n",
       "      <td>-0.219091</td>\n",
       "      <td>-0.300909</td>\n",
       "      <td>-0.480455</td>\n",
       "      <td>-0.176364</td>\n",
       "      <td>-0.395455</td>\n",
       "      <td>-0.378636</td>\n",
       "      <td>-0.171364</td>\n",
       "      <td>-0.3980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.229091</td>\n",
       "      <td>-0.280455</td>\n",
       "      <td>-0.359091</td>\n",
       "      <td>-0.239091</td>\n",
       "      <td>-0.330455</td>\n",
       "      <td>-0.282727</td>\n",
       "      <td>-0.260909</td>\n",
       "      <td>-0.334545</td>\n",
       "      <td>-0.233636</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.185455</td>\n",
       "      <td>-0.294091</td>\n",
       "      <td>-0.118636</td>\n",
       "      <td>-0.200455</td>\n",
       "      <td>-0.246364</td>\n",
       "      <td>-0.138182</td>\n",
       "      <td>-0.199545</td>\n",
       "      <td>-0.198182</td>\n",
       "      <td>-0.132273</td>\n",
       "      <td>-0.2860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.170000</td>\n",
       "      <td>-0.194545</td>\n",
       "      <td>-0.297727</td>\n",
       "      <td>-0.155000</td>\n",
       "      <td>-0.197727</td>\n",
       "      <td>-0.313636</td>\n",
       "      <td>-0.147273</td>\n",
       "      <td>-0.225455</td>\n",
       "      <td>-0.321364</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.256818</td>\n",
       "      <td>-0.233182</td>\n",
       "      <td>-0.388182</td>\n",
       "      <td>-0.248636</td>\n",
       "      <td>-0.254091</td>\n",
       "      <td>-0.356818</td>\n",
       "      <td>-0.215000</td>\n",
       "      <td>-0.304091</td>\n",
       "      <td>-0.347273</td>\n",
       "      <td>-0.2045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14040</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.292273</td>\n",
       "      <td>-0.344091</td>\n",
       "      <td>-0.269091</td>\n",
       "      <td>-0.230909</td>\n",
       "      <td>-0.333636</td>\n",
       "      <td>-0.200455</td>\n",
       "      <td>-0.248636</td>\n",
       "      <td>-0.304545</td>\n",
       "      <td>-0.186818</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.201818</td>\n",
       "      <td>-0.217727</td>\n",
       "      <td>-0.198182</td>\n",
       "      <td>-0.176364</td>\n",
       "      <td>-0.194091</td>\n",
       "      <td>-0.173182</td>\n",
       "      <td>-0.170000</td>\n",
       "      <td>-0.163636</td>\n",
       "      <td>-0.173636</td>\n",
       "      <td>-0.1470</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14041 rows × 1501 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       af         0         1         2         3         4         5  \\\n",
       "0       0 -0.298182 -0.288636 -0.152727 -0.203182 -0.275909 -0.091818   \n",
       "1       0 -0.308182 -0.372727  0.012727  0.195909  0.140000  0.100455   \n",
       "2       0 -0.240909 -0.326818 -0.219091 -0.444091 -0.686818 -0.429091   \n",
       "3       0 -0.935000 -0.363636 -0.209091 -0.270000 -0.517727 -0.579091   \n",
       "4       0 -0.493636 -0.509091 -0.511818 -0.500455 -0.495455 -0.451818   \n",
       "...    ..       ...       ...       ...       ...       ...       ...   \n",
       "14036   1 -0.266818 -0.125000 -0.283636 -0.163182 -0.210000 -0.207273   \n",
       "14037   1 -0.526818 -0.296364 -0.358182 -0.539091 -0.261818 -0.445455   \n",
       "14038   1 -0.229091 -0.280455 -0.359091 -0.239091 -0.330455 -0.282727   \n",
       "14039   1 -0.170000 -0.194545 -0.297727 -0.155000 -0.197727 -0.313636   \n",
       "14040   1 -0.292273 -0.344091 -0.269091 -0.230909 -0.333636 -0.200455   \n",
       "\n",
       "              6         7         8  ...      1490      1491      1492  \\\n",
       "0     -0.853182 -0.425909 -0.460909  ...  0.398636  0.518182  0.453182   \n",
       "1      0.054091  0.027727  0.025909  ... -0.787727 -0.485000 -0.438182   \n",
       "2     -0.436818 -0.509091 -0.717273  ... -0.124091 -0.152273 -0.293636   \n",
       "3     -0.535909 -0.272727 -0.157727  ... -0.634091 -0.386364 -0.332273   \n",
       "4     -0.267727 -0.538636 -0.310909  ... -0.727727 -0.699545 -0.652273   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "14036 -0.259545 -0.276364 -0.111364  ... -0.385909 -0.204545 -0.520909   \n",
       "14037 -0.464091 -0.241818 -0.518636  ... -0.213636 -0.496818 -0.219091   \n",
       "14038 -0.260909 -0.334545 -0.233636  ... -0.185455 -0.294091 -0.118636   \n",
       "14039 -0.147273 -0.225455 -0.321364  ... -0.256818 -0.233182 -0.388182   \n",
       "14040 -0.248636 -0.304545 -0.186818  ... -0.201818 -0.217727 -0.198182   \n",
       "\n",
       "           1493      1494      1495      1496      1497      1498    1499  \n",
       "0      0.409091  0.640909 -0.179545  0.252727  0.228636  0.143182 -0.0355  \n",
       "1     -0.430909 -0.452273 -0.454545 -0.445909 -0.428182 -0.414545 -0.3965  \n",
       "2     -0.414091 -0.443182 -0.370455 -0.036818 -0.098636 -0.185000  0.1520  \n",
       "3     -0.392273 -0.492727 -0.485455 -0.477727 -0.489545 -0.484091 -0.4805  \n",
       "4     -0.595000 -0.542273 -0.493636 -0.468636 -0.451364 -0.437727 -0.4220  \n",
       "...         ...       ...       ...       ...       ...       ...     ...  \n",
       "14036 -0.342727 -0.254545 -0.534091 -0.207727 -0.458182 -0.441818 -0.2315  \n",
       "14037 -0.300909 -0.480455 -0.176364 -0.395455 -0.378636 -0.171364 -0.3980  \n",
       "14038 -0.200455 -0.246364 -0.138182 -0.199545 -0.198182 -0.132273 -0.2860  \n",
       "14039 -0.248636 -0.254091 -0.356818 -0.215000 -0.304091 -0.347273 -0.2045  \n",
       "14040 -0.176364 -0.194091 -0.173182 -0.170000 -0.163636 -0.173636 -0.1470  \n",
       "\n",
       "[14041 rows x 1501 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "newdf_list = list()\n",
    "af_column = df.af.copy()\n",
    "for i in range(1, len(df.columns)-9, 10):\n",
    "#     print((df[i:i+10]).mean(axis = 1)); break\n",
    "    newdf_list.append((df[df.columns[i:i+11]]).mean(axis = 1))\n",
    "\n",
    "newdf4 = pd.concat(newdf_list, axis = 1)\n",
    "newdf4.insert(0, 'af', af_column)\n",
    "newdf4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "newdf4.to_csv('/hdd/physio/af/hugedfby10.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf4.isna().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "newdf4 = pd.read_csv('/hdd/physio/af/hugedfby10.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>1393</th>\n",
       "      <th>1394</th>\n",
       "      <th>1395</th>\n",
       "      <th>1396</th>\n",
       "      <th>1397</th>\n",
       "      <th>1398</th>\n",
       "      <th>1399</th>\n",
       "      <th>1400</th>\n",
       "      <th>1401</th>\n",
       "      <th>1402</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.332846</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.244549</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.272272</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.304225</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.233769</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14025</th>\n",
       "      <td>14026</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.286884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14026</th>\n",
       "      <td>14027</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.254939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14027</th>\n",
       "      <td>14028</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.352475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14028</th>\n",
       "      <td>14029</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.297423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14029</th>\n",
       "      <td>14030</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.187688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14030 rows × 1405 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  af         0   1   2   3   4   5   6   7  ...  1393  1394  \\\n",
       "0               1   0 -0.332846 NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "1               2   0 -0.244549 NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "2               3   0 -0.272272 NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "3               4   0 -0.304225 NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "4               5   0 -0.233769 NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "...           ...  ..       ...  ..  ..  ..  ..  ..  ..  ..  ...   ...   ...   \n",
       "14025       14026   1       NaN NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "14026       14027   1       NaN NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "14027       14028   1       NaN NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "14028       14029   1       NaN NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "14029       14030   1       NaN NaN NaN NaN NaN NaN NaN NaN  ...   NaN   NaN   \n",
       "\n",
       "       1395  1396  1397  1398  1399  1400  1401      1402  \n",
       "0       NaN   NaN   NaN   NaN   NaN   NaN   NaN       NaN  \n",
       "1       NaN   NaN   NaN   NaN   NaN   NaN   NaN       NaN  \n",
       "2       NaN   NaN   NaN   NaN   NaN   NaN   NaN       NaN  \n",
       "3       NaN   NaN   NaN   NaN   NaN   NaN   NaN       NaN  \n",
       "4       NaN   NaN   NaN   NaN   NaN   NaN   NaN       NaN  \n",
       "...     ...   ...   ...   ...   ...   ...   ...       ...  \n",
       "14025   NaN   NaN   NaN   NaN   NaN   NaN   NaN -0.286884  \n",
       "14026   NaN   NaN   NaN   NaN   NaN   NaN   NaN -0.254939  \n",
       "14027   NaN   NaN   NaN   NaN   NaN   NaN   NaN -0.352475  \n",
       "14028   NaN   NaN   NaN   NaN   NaN   NaN   NaN -0.297423  \n",
       "14029   NaN   NaN   NaN   NaN   NaN   NaN   NaN -0.187688  \n",
       "\n",
       "[14030 rows x 1405 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>14990</th>\n",
       "      <th>14991</th>\n",
       "      <th>14992</th>\n",
       "      <th>14993</th>\n",
       "      <th>14994</th>\n",
       "      <th>14995</th>\n",
       "      <th>14996</th>\n",
       "      <th>14997</th>\n",
       "      <th>14998</th>\n",
       "      <th>14999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.275</td>\n",
       "      <td>-0.295</td>\n",
       "      <td>-0.310</td>\n",
       "      <td>-0.315</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.320</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.290</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>-0.025</td>\n",
       "      <td>-0.020</td>\n",
       "      <td>-0.055</td>\n",
       "      <td>-0.075</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.195</td>\n",
       "      <td>-0.220</td>\n",
       "      <td>-0.245</td>\n",
       "      <td>-0.250</td>\n",
       "      <td>-0.315</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.315</td>\n",
       "      <td>-0.350</td>\n",
       "      <td>-0.380</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.425</td>\n",
       "      <td>-0.395</td>\n",
       "      <td>-0.395</td>\n",
       "      <td>-0.385</td>\n",
       "      <td>-0.405</td>\n",
       "      <td>-0.445</td>\n",
       "      <td>-0.415</td>\n",
       "      <td>-0.420</td>\n",
       "      <td>-0.360</td>\n",
       "      <td>-0.320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.285</td>\n",
       "      <td>-0.245</td>\n",
       "      <td>-0.270</td>\n",
       "      <td>-0.250</td>\n",
       "      <td>-0.230</td>\n",
       "      <td>-0.215</td>\n",
       "      <td>-0.215</td>\n",
       "      <td>-0.225</td>\n",
       "      <td>-0.260</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.320</td>\n",
       "      <td>-0.265</td>\n",
       "      <td>-0.185</td>\n",
       "      <td>-0.120</td>\n",
       "      <td>-0.005</td>\n",
       "      <td>0.165</td>\n",
       "      <td>0.410</td>\n",
       "      <td>0.590</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.925</td>\n",
       "      <td>-1.560</td>\n",
       "      <td>-1.800</td>\n",
       "      <td>-1.705</td>\n",
       "      <td>-1.335</td>\n",
       "      <td>-0.945</td>\n",
       "      <td>-0.665</td>\n",
       "      <td>-0.510</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.495</td>\n",
       "      <td>-0.490</td>\n",
       "      <td>-0.445</td>\n",
       "      <td>-0.420</td>\n",
       "      <td>-0.450</td>\n",
       "      <td>-0.455</td>\n",
       "      <td>-0.520</td>\n",
       "      <td>-0.550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.560</td>\n",
       "      <td>-0.540</td>\n",
       "      <td>-0.530</td>\n",
       "      <td>-0.505</td>\n",
       "      <td>-0.505</td>\n",
       "      <td>-0.465</td>\n",
       "      <td>-0.465</td>\n",
       "      <td>-0.480</td>\n",
       "      <td>-0.440</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.375</td>\n",
       "      <td>-0.380</td>\n",
       "      <td>-0.390</td>\n",
       "      <td>-0.405</td>\n",
       "      <td>-0.425</td>\n",
       "      <td>-0.440</td>\n",
       "      <td>-0.465</td>\n",
       "      <td>-0.450</td>\n",
       "      <td>-0.450</td>\n",
       "      <td>-0.440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.295</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.110</td>\n",
       "      <td>-0.485</td>\n",
       "      <td>-0.935</td>\n",
       "      <td>-0.690</td>\n",
       "      <td>-0.115</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.360</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.060</td>\n",
       "      <td>0.325</td>\n",
       "      <td>-0.010</td>\n",
       "      <td>-0.620</td>\n",
       "      <td>-0.615</td>\n",
       "      <td>-0.190</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.595</td>\n",
       "      <td>-0.305</td>\n",
       "      <td>0.055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.230</td>\n",
       "      <td>-0.890</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-0.430</td>\n",
       "      <td>-0.105</td>\n",
       "      <td>-0.420</td>\n",
       "      <td>-0.550</td>\n",
       "      <td>-0.200</td>\n",
       "      <td>-0.245</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.090</td>\n",
       "      <td>-0.720</td>\n",
       "      <td>-1.060</td>\n",
       "      <td>-0.715</td>\n",
       "      <td>-0.140</td>\n",
       "      <td>-0.110</td>\n",
       "      <td>-0.430</td>\n",
       "      <td>-0.375</td>\n",
       "      <td>-0.060</td>\n",
       "      <td>-0.280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.665</td>\n",
       "      <td>-0.500</td>\n",
       "      <td>0.110</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.030</td>\n",
       "      <td>-0.460</td>\n",
       "      <td>-0.385</td>\n",
       "      <td>-0.080</td>\n",
       "      <td>-0.270</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.160</td>\n",
       "      <td>-0.400</td>\n",
       "      <td>-0.905</td>\n",
       "      <td>-0.705</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.070</td>\n",
       "      <td>-0.400</td>\n",
       "      <td>-0.250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>1</td>\n",
       "      <td>0.075</td>\n",
       "      <td>-0.190</td>\n",
       "      <td>-0.695</td>\n",
       "      <td>-0.660</td>\n",
       "      <td>-0.005</td>\n",
       "      <td>0.395</td>\n",
       "      <td>0.115</td>\n",
       "      <td>-0.365</td>\n",
       "      <td>-0.265</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.425</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.130</td>\n",
       "      <td>-0.420</td>\n",
       "      <td>-0.660</td>\n",
       "      <td>-0.285</td>\n",
       "      <td>0.265</td>\n",
       "      <td>0.265</td>\n",
       "      <td>-0.230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14040</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.480</td>\n",
       "      <td>-0.220</td>\n",
       "      <td>-0.140</td>\n",
       "      <td>-0.465</td>\n",
       "      <td>-0.415</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.215</td>\n",
       "      <td>-0.240</td>\n",
       "      <td>-0.730</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.145</td>\n",
       "      <td>-0.145</td>\n",
       "      <td>-0.180</td>\n",
       "      <td>-0.110</td>\n",
       "      <td>-0.075</td>\n",
       "      <td>-0.095</td>\n",
       "      <td>-0.170</td>\n",
       "      <td>-0.210</td>\n",
       "      <td>-0.180</td>\n",
       "      <td>-0.160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14041 rows × 15001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       af      0      1      2      3      4      5      6      7      8  ...  \\\n",
       "0       0 -0.275 -0.295 -0.310 -0.315 -0.300 -0.320 -0.300 -0.300 -0.290  ...   \n",
       "1       0 -0.195 -0.220 -0.245 -0.250 -0.315 -0.300 -0.315 -0.350 -0.380  ...   \n",
       "2       0 -0.285 -0.245 -0.270 -0.250 -0.230 -0.215 -0.215 -0.225 -0.260  ...   \n",
       "3       0  0.000 -0.925 -1.560 -1.800 -1.705 -1.335 -0.945 -0.665 -0.510  ...   \n",
       "4       0 -0.560 -0.540 -0.530 -0.505 -0.505 -0.465 -0.465 -0.480 -0.440  ...   \n",
       "...    ..    ...    ...    ...    ...    ...    ...    ...    ...    ...  ...   \n",
       "14036   1 -0.295  0.140  0.110 -0.485 -0.935 -0.690 -0.115  0.000 -0.360  ...   \n",
       "14037   1 -0.230 -0.890 -1.000 -0.430 -0.105 -0.420 -0.550 -0.200 -0.245  ...   \n",
       "14038   1 -0.665 -0.500  0.110  0.400  0.030 -0.460 -0.385 -0.080 -0.270  ...   \n",
       "14039   1  0.075 -0.190 -0.695 -0.660 -0.005  0.395  0.115 -0.365 -0.265  ...   \n",
       "14040   1 -0.480 -0.220 -0.140 -0.465 -0.415  0.050  0.215 -0.240 -0.730  ...   \n",
       "\n",
       "       14990  14991  14992  14993  14994  14995  14996  14997  14998  14999  \n",
       "0      0.070  0.015  0.005  0.005 -0.025 -0.020 -0.055 -0.075 -0.125 -0.150  \n",
       "1     -0.425 -0.395 -0.395 -0.385 -0.405 -0.445 -0.415 -0.420 -0.360 -0.320  \n",
       "2     -0.320 -0.265 -0.185 -0.120 -0.005  0.165  0.410  0.590  0.660  0.590  \n",
       "3     -0.485 -0.495 -0.495 -0.490 -0.445 -0.420 -0.450 -0.455 -0.520 -0.550  \n",
       "4     -0.375 -0.380 -0.390 -0.405 -0.425 -0.440 -0.465 -0.450 -0.450 -0.440  \n",
       "...      ...    ...    ...    ...    ...    ...    ...    ...    ...    ...  \n",
       "14036 -0.060  0.325 -0.010 -0.620 -0.615 -0.190 -0.300 -0.595 -0.305  0.055  \n",
       "14037 -0.090 -0.720 -1.060 -0.715 -0.140 -0.110 -0.430 -0.375 -0.060 -0.280  \n",
       "14038 -0.300  0.135  0.160 -0.400 -0.905 -0.705 -0.125 -0.070 -0.400 -0.250  \n",
       "14039 -0.125 -0.425 -0.300 -0.130 -0.420 -0.660 -0.285  0.265  0.265 -0.230  \n",
       "14040 -0.145 -0.145 -0.180 -0.110 -0.075 -0.095 -0.170 -0.210 -0.180 -0.160  \n",
       "\n",
       "[14041 rows x 15001 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(how = 'any', inplace = True)\n",
    "df.reset_index(drop = True, inplace = True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-00cf07b74dcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(newdf3, test_size=0.2, random_state=42, shuffle=True)\n",
    "\n",
    "train = train.to_numpy()\n",
    "test = test.to_numpy()\n",
    "\n",
    "# path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try7\"\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/FastCells/try7-af\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import TomekLinks\n",
    "tomek = TomekLinks()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.12 s, sys: 188 ms, total: 5.31 s\n",
      "Wall time: 6.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df = pd.read_csv('/hdd/physio/af/hugedfby10.csv', index_col = 0)\n",
    "df\n",
    "\n",
    "train, test = train_test_split(df, test_size=0.2, random_state=42, shuffle=True)\n",
    "\n",
    "train = train.to_numpy()\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/FastCells/tryaffast\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>af</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>1490</th>\n",
       "      <th>1491</th>\n",
       "      <th>1492</th>\n",
       "      <th>1493</th>\n",
       "      <th>1494</th>\n",
       "      <th>1495</th>\n",
       "      <th>1496</th>\n",
       "      <th>1497</th>\n",
       "      <th>1498</th>\n",
       "      <th>1499</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.298182</td>\n",
       "      <td>-0.288636</td>\n",
       "      <td>-0.152727</td>\n",
       "      <td>-0.203182</td>\n",
       "      <td>-0.275909</td>\n",
       "      <td>-0.091818</td>\n",
       "      <td>-0.853182</td>\n",
       "      <td>-0.425909</td>\n",
       "      <td>-0.460909</td>\n",
       "      <td>...</td>\n",
       "      <td>0.398636</td>\n",
       "      <td>0.518182</td>\n",
       "      <td>0.453182</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.640909</td>\n",
       "      <td>-0.179545</td>\n",
       "      <td>0.252727</td>\n",
       "      <td>0.228636</td>\n",
       "      <td>0.143182</td>\n",
       "      <td>-0.0355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.308182</td>\n",
       "      <td>-0.372727</td>\n",
       "      <td>0.012727</td>\n",
       "      <td>0.195909</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.100455</td>\n",
       "      <td>0.054091</td>\n",
       "      <td>0.027727</td>\n",
       "      <td>0.025909</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.787727</td>\n",
       "      <td>-0.485000</td>\n",
       "      <td>-0.438182</td>\n",
       "      <td>-0.430909</td>\n",
       "      <td>-0.452273</td>\n",
       "      <td>-0.454545</td>\n",
       "      <td>-0.445909</td>\n",
       "      <td>-0.428182</td>\n",
       "      <td>-0.414545</td>\n",
       "      <td>-0.3965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.240909</td>\n",
       "      <td>-0.326818</td>\n",
       "      <td>-0.219091</td>\n",
       "      <td>-0.444091</td>\n",
       "      <td>-0.686818</td>\n",
       "      <td>-0.429091</td>\n",
       "      <td>-0.436818</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.717273</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.124091</td>\n",
       "      <td>-0.152273</td>\n",
       "      <td>-0.293636</td>\n",
       "      <td>-0.414091</td>\n",
       "      <td>-0.443182</td>\n",
       "      <td>-0.370455</td>\n",
       "      <td>-0.036818</td>\n",
       "      <td>-0.098636</td>\n",
       "      <td>-0.185000</td>\n",
       "      <td>0.1520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.935000</td>\n",
       "      <td>-0.363636</td>\n",
       "      <td>-0.209091</td>\n",
       "      <td>-0.270000</td>\n",
       "      <td>-0.517727</td>\n",
       "      <td>-0.579091</td>\n",
       "      <td>-0.535909</td>\n",
       "      <td>-0.272727</td>\n",
       "      <td>-0.157727</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.634091</td>\n",
       "      <td>-0.386364</td>\n",
       "      <td>-0.332273</td>\n",
       "      <td>-0.392273</td>\n",
       "      <td>-0.492727</td>\n",
       "      <td>-0.485455</td>\n",
       "      <td>-0.477727</td>\n",
       "      <td>-0.489545</td>\n",
       "      <td>-0.484091</td>\n",
       "      <td>-0.4805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.493636</td>\n",
       "      <td>-0.509091</td>\n",
       "      <td>-0.511818</td>\n",
       "      <td>-0.500455</td>\n",
       "      <td>-0.495455</td>\n",
       "      <td>-0.451818</td>\n",
       "      <td>-0.267727</td>\n",
       "      <td>-0.538636</td>\n",
       "      <td>-0.310909</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.727727</td>\n",
       "      <td>-0.699545</td>\n",
       "      <td>-0.652273</td>\n",
       "      <td>-0.595000</td>\n",
       "      <td>-0.542273</td>\n",
       "      <td>-0.493636</td>\n",
       "      <td>-0.468636</td>\n",
       "      <td>-0.451364</td>\n",
       "      <td>-0.437727</td>\n",
       "      <td>-0.4220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.266818</td>\n",
       "      <td>-0.125000</td>\n",
       "      <td>-0.283636</td>\n",
       "      <td>-0.163182</td>\n",
       "      <td>-0.210000</td>\n",
       "      <td>-0.207273</td>\n",
       "      <td>-0.259545</td>\n",
       "      <td>-0.276364</td>\n",
       "      <td>-0.111364</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.385909</td>\n",
       "      <td>-0.204545</td>\n",
       "      <td>-0.520909</td>\n",
       "      <td>-0.342727</td>\n",
       "      <td>-0.254545</td>\n",
       "      <td>-0.534091</td>\n",
       "      <td>-0.207727</td>\n",
       "      <td>-0.458182</td>\n",
       "      <td>-0.441818</td>\n",
       "      <td>-0.2315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.526818</td>\n",
       "      <td>-0.296364</td>\n",
       "      <td>-0.358182</td>\n",
       "      <td>-0.539091</td>\n",
       "      <td>-0.261818</td>\n",
       "      <td>-0.445455</td>\n",
       "      <td>-0.464091</td>\n",
       "      <td>-0.241818</td>\n",
       "      <td>-0.518636</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.213636</td>\n",
       "      <td>-0.496818</td>\n",
       "      <td>-0.219091</td>\n",
       "      <td>-0.300909</td>\n",
       "      <td>-0.480455</td>\n",
       "      <td>-0.176364</td>\n",
       "      <td>-0.395455</td>\n",
       "      <td>-0.378636</td>\n",
       "      <td>-0.171364</td>\n",
       "      <td>-0.3980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.229091</td>\n",
       "      <td>-0.280455</td>\n",
       "      <td>-0.359091</td>\n",
       "      <td>-0.239091</td>\n",
       "      <td>-0.330455</td>\n",
       "      <td>-0.282727</td>\n",
       "      <td>-0.260909</td>\n",
       "      <td>-0.334545</td>\n",
       "      <td>-0.233636</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.185455</td>\n",
       "      <td>-0.294091</td>\n",
       "      <td>-0.118636</td>\n",
       "      <td>-0.200455</td>\n",
       "      <td>-0.246364</td>\n",
       "      <td>-0.138182</td>\n",
       "      <td>-0.199545</td>\n",
       "      <td>-0.198182</td>\n",
       "      <td>-0.132273</td>\n",
       "      <td>-0.2860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.170000</td>\n",
       "      <td>-0.194545</td>\n",
       "      <td>-0.297727</td>\n",
       "      <td>-0.155000</td>\n",
       "      <td>-0.197727</td>\n",
       "      <td>-0.313636</td>\n",
       "      <td>-0.147273</td>\n",
       "      <td>-0.225455</td>\n",
       "      <td>-0.321364</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.256818</td>\n",
       "      <td>-0.233182</td>\n",
       "      <td>-0.388182</td>\n",
       "      <td>-0.248636</td>\n",
       "      <td>-0.254091</td>\n",
       "      <td>-0.356818</td>\n",
       "      <td>-0.215000</td>\n",
       "      <td>-0.304091</td>\n",
       "      <td>-0.347273</td>\n",
       "      <td>-0.2045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14040</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.292273</td>\n",
       "      <td>-0.344091</td>\n",
       "      <td>-0.269091</td>\n",
       "      <td>-0.230909</td>\n",
       "      <td>-0.333636</td>\n",
       "      <td>-0.200455</td>\n",
       "      <td>-0.248636</td>\n",
       "      <td>-0.304545</td>\n",
       "      <td>-0.186818</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.201818</td>\n",
       "      <td>-0.217727</td>\n",
       "      <td>-0.198182</td>\n",
       "      <td>-0.176364</td>\n",
       "      <td>-0.194091</td>\n",
       "      <td>-0.173182</td>\n",
       "      <td>-0.170000</td>\n",
       "      <td>-0.163636</td>\n",
       "      <td>-0.173636</td>\n",
       "      <td>-0.1470</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14041 rows × 1501 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       af         0         1         2         3         4         5  \\\n",
       "0       0 -0.298182 -0.288636 -0.152727 -0.203182 -0.275909 -0.091818   \n",
       "1       0 -0.308182 -0.372727  0.012727  0.195909  0.140000  0.100455   \n",
       "2       0 -0.240909 -0.326818 -0.219091 -0.444091 -0.686818 -0.429091   \n",
       "3       0 -0.935000 -0.363636 -0.209091 -0.270000 -0.517727 -0.579091   \n",
       "4       0 -0.493636 -0.509091 -0.511818 -0.500455 -0.495455 -0.451818   \n",
       "...    ..       ...       ...       ...       ...       ...       ...   \n",
       "14036   1 -0.266818 -0.125000 -0.283636 -0.163182 -0.210000 -0.207273   \n",
       "14037   1 -0.526818 -0.296364 -0.358182 -0.539091 -0.261818 -0.445455   \n",
       "14038   1 -0.229091 -0.280455 -0.359091 -0.239091 -0.330455 -0.282727   \n",
       "14039   1 -0.170000 -0.194545 -0.297727 -0.155000 -0.197727 -0.313636   \n",
       "14040   1 -0.292273 -0.344091 -0.269091 -0.230909 -0.333636 -0.200455   \n",
       "\n",
       "              6         7         8  ...      1490      1491      1492  \\\n",
       "0     -0.853182 -0.425909 -0.460909  ...  0.398636  0.518182  0.453182   \n",
       "1      0.054091  0.027727  0.025909  ... -0.787727 -0.485000 -0.438182   \n",
       "2     -0.436818 -0.509091 -0.717273  ... -0.124091 -0.152273 -0.293636   \n",
       "3     -0.535909 -0.272727 -0.157727  ... -0.634091 -0.386364 -0.332273   \n",
       "4     -0.267727 -0.538636 -0.310909  ... -0.727727 -0.699545 -0.652273   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "14036 -0.259545 -0.276364 -0.111364  ... -0.385909 -0.204545 -0.520909   \n",
       "14037 -0.464091 -0.241818 -0.518636  ... -0.213636 -0.496818 -0.219091   \n",
       "14038 -0.260909 -0.334545 -0.233636  ... -0.185455 -0.294091 -0.118636   \n",
       "14039 -0.147273 -0.225455 -0.321364  ... -0.256818 -0.233182 -0.388182   \n",
       "14040 -0.248636 -0.304545 -0.186818  ... -0.201818 -0.217727 -0.198182   \n",
       "\n",
       "           1493      1494      1495      1496      1497      1498    1499  \n",
       "0      0.409091  0.640909 -0.179545  0.252727  0.228636  0.143182 -0.0355  \n",
       "1     -0.430909 -0.452273 -0.454545 -0.445909 -0.428182 -0.414545 -0.3965  \n",
       "2     -0.414091 -0.443182 -0.370455 -0.036818 -0.098636 -0.185000  0.1520  \n",
       "3     -0.392273 -0.492727 -0.485455 -0.477727 -0.489545 -0.484091 -0.4805  \n",
       "4     -0.595000 -0.542273 -0.493636 -0.468636 -0.451364 -0.437727 -0.4220  \n",
       "...         ...       ...       ...       ...       ...       ...     ...  \n",
       "14036 -0.342727 -0.254545 -0.534091 -0.207727 -0.458182 -0.441818 -0.2315  \n",
       "14037 -0.300909 -0.480455 -0.176364 -0.395455 -0.378636 -0.171364 -0.3980  \n",
       "14038 -0.200455 -0.246364 -0.138182 -0.199545 -0.198182 -0.132273 -0.2860  \n",
       "14039 -0.248636 -0.254091 -0.356818 -0.215000 -0.304091 -0.347273 -0.2045  \n",
       "14040 -0.176364 -0.194091 -0.173182 -0.170000 -0.163636 -0.173636 -0.1470  \n",
       "\n",
       "[14041 rows x 1501 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><ins>VF+AF</ins></h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 49.8 s, sys: 812 ms, total: 50.6 s\n",
      "Wall time: 52.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "afdf = pd.read_csv('/hdd/physio/af/rpeakdf.csv', index_col = 0)\n",
    "vfd = pd.read_csv(\"/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv\", index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 32.7 s, sys: 213 ms, total: 32.9 s\n",
      "Wall time: 32.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def featureExtraction(df, fs):\n",
    "    df1 = df.af.copy()\n",
    "    df.drop(columns = ['af'], inplace = True)\n",
    "    final = list()\n",
    "    for i in range(len(df)):\n",
    "        peaks = []\n",
    "        peaks = df.loc[i][df.loc[i] == 1].index.values\n",
    "        peaks = [int(x) for x in peaks]\n",
    "        if (len(peaks) == 0 or len(peaks) == 1):\n",
    "            continue\n",
    "        af = df1.loc[i]\n",
    "        rr = np.diff(peaks) / fs \n",
    "        hr = 60 / rr\n",
    "        rmssd = np.sqrt(np.mean(np.square(rr)))\n",
    "        sdnn = np.std(rr)\n",
    "        mean_rr = np.mean(rr)\n",
    "        mean_hr = np.mean(hr)\n",
    "        std_hr = np.std(hr)\n",
    "        min_hr = np.min(hr)\n",
    "        max_hr = np.max(hr)\n",
    "        single_window_df = pd.DataFrame([[rmssd, sdnn, mean_rr, mean_hr, std_hr, min_hr, max_hr]], columns = ['RMSSD', 'STDNN', 'MEAN_RR', 'MEAN_HR', 'STD_HR', 'MIN_HR', 'MAX_HR'])\n",
    "        single_window_df.insert(0, 'ground_truth', af)\n",
    "        final.append(single_window_df)\n",
    "        del single_window_df\n",
    "    return pd.concat(final)\n",
    "\n",
    "# afd = featureExtraction(afdf, 250)\n",
    "# afd.reset_index(drop = True, inplace = True)\n",
    "vfd = featureExtraction(vfdf, 250)\n",
    "vfd.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ground_truth</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ground_truth     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR  \\\n",
       "0                0  1.081660  0.026600  1.081333   55.521205   1.388378   \n",
       "1                0  1.048015  0.005657  1.048000   57.253570   0.307866   \n",
       "2                0  1.065197  0.020469  1.065000   56.358432   1.061594   \n",
       "3                0  1.066197  0.020494  1.066000   56.305640   1.064400   \n",
       "4                0  1.070828  0.018571  1.070667   56.056575   0.964208   \n",
       "...            ...       ...       ...       ...         ...        ...   \n",
       "3723             1  0.887601  0.410447  0.787000  101.853431  50.658017   \n",
       "3724             1  1.382036  0.010000  1.382000   43.417613   0.314165   \n",
       "3725             1  1.378703  0.009978  1.378667   43.522583   0.314125   \n",
       "3726             1  1.378672  0.003771  1.378667   43.520634   0.118818   \n",
       "3727             1  1.393358  0.008219  1.393333   43.063697   0.253679   \n",
       "\n",
       "         MIN_HR      MAX_HR  \n",
       "0     54.347826   57.471264  \n",
       "1     56.818182   57.471264  \n",
       "2     54.545455   57.251908  \n",
       "3     54.545455   57.251908  \n",
       "4     54.744526   57.034221  \n",
       "...         ...         ...  \n",
       "3723  43.988270  164.835165  \n",
       "3724  43.103448   43.731778  \n",
       "3725  43.103448   43.859649  \n",
       "3726  43.352601   43.604651  \n",
       "3727  42.735043   43.352601  \n",
       "\n",
       "[3728 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vfd.rename(columns = {'vf': 'ground_truth'}, inplace = True)\n",
    "vfd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ground_truth</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.720403</td>\n",
       "      <td>0.199647</td>\n",
       "      <td>0.692186</td>\n",
       "      <td>95.439177</td>\n",
       "      <td>31.962709</td>\n",
       "      <td>52.264808</td>\n",
       "      <td>202.702703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.685475</td>\n",
       "      <td>0.209389</td>\n",
       "      <td>0.652711</td>\n",
       "      <td>101.701272</td>\n",
       "      <td>33.196907</td>\n",
       "      <td>50.675676</td>\n",
       "      <td>254.237288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.718068</td>\n",
       "      <td>0.221976</td>\n",
       "      <td>0.682897</td>\n",
       "      <td>96.715793</td>\n",
       "      <td>28.657022</td>\n",
       "      <td>49.342105</td>\n",
       "      <td>170.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.624761</td>\n",
       "      <td>0.193625</td>\n",
       "      <td>0.594000</td>\n",
       "      <td>112.472761</td>\n",
       "      <td>36.208551</td>\n",
       "      <td>60.728745</td>\n",
       "      <td>194.805195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.730429</td>\n",
       "      <td>0.230748</td>\n",
       "      <td>0.693023</td>\n",
       "      <td>95.938086</td>\n",
       "      <td>29.934269</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>192.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>1</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>187.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>187.500000</td>\n",
       "      <td>187.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>0</td>\n",
       "      <td>0.730280</td>\n",
       "      <td>0.065540</td>\n",
       "      <td>0.727333</td>\n",
       "      <td>83.143787</td>\n",
       "      <td>7.259160</td>\n",
       "      <td>70.754717</td>\n",
       "      <td>94.339623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>0</td>\n",
       "      <td>0.698771</td>\n",
       "      <td>0.038418</td>\n",
       "      <td>0.697714</td>\n",
       "      <td>86.258796</td>\n",
       "      <td>4.795131</td>\n",
       "      <td>79.787234</td>\n",
       "      <td>94.339623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>0</td>\n",
       "      <td>0.696291</td>\n",
       "      <td>0.020133</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>86.279357</td>\n",
       "      <td>2.505909</td>\n",
       "      <td>82.872928</td>\n",
       "      <td>90.361446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>0</td>\n",
       "      <td>0.696184</td>\n",
       "      <td>0.016000</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>86.252949</td>\n",
       "      <td>2.003812</td>\n",
       "      <td>83.798883</td>\n",
       "      <td>89.820359</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15741 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ground_truth     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR  \\\n",
       "0                0  0.720403  0.199647  0.692186   95.439177  31.962709   \n",
       "1                0  0.685475  0.209389  0.652711  101.701272  33.196907   \n",
       "2                0  0.718068  0.221976  0.682897   96.715793  28.657022   \n",
       "3                0  0.624761  0.193625  0.594000  112.472761  36.208551   \n",
       "4                0  0.730429  0.230748  0.693023   95.938086  29.934269   \n",
       "...            ...       ...       ...       ...         ...        ...   \n",
       "1995             1  0.320000  0.000000  0.320000  187.500000   0.000000   \n",
       "1996             0  0.730280  0.065540  0.727333   83.143787   7.259160   \n",
       "1997             0  0.698771  0.038418  0.697714   86.258796   4.795131   \n",
       "1998             0  0.696291  0.020133  0.696000   86.279357   2.505909   \n",
       "1999             0  0.696184  0.016000  0.696000   86.252949   2.003812   \n",
       "\n",
       "          MIN_HR      MAX_HR  \n",
       "0      52.264808  202.702703  \n",
       "1      50.675676  254.237288  \n",
       "2      49.342105  170.454545  \n",
       "3      60.728745  194.805195  \n",
       "4      43.859649  192.307692  \n",
       "...          ...         ...  \n",
       "1995  187.500000  187.500000  \n",
       "1996   70.754717   94.339623  \n",
       "1997   79.787234   94.339623  \n",
       "1998   82.872928   90.361446  \n",
       "1999   83.798883   89.820359  \n",
       "\n",
       "[15741 rows x 8 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.concat([afd, vfd[:2000]], axis = 0)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = train_test_split(data, test_size=0.2, random_state=42, shuffle=True)\n",
    "\n",
    "# train = data\n",
    "# train = train.to_numpy()\n",
    "test = vfd[2000:]\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try2vfaf\"\n",
    "\n",
    "# np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR  \n",
       "0      57.471264  \n",
       "1      57.471264  \n",
       "2      57.251908  \n",
       "3      57.251908  \n",
       "4      57.034221  \n",
       "...          ...  \n",
       "3723  164.835165  \n",
       "3724   43.731778  \n",
       "3725   43.859649  \n",
       "3726   43.604651  \n",
       "3727   43.352601  \n",
       "\n",
       "[3728 rows x 8 columns]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame = pd.read_csv('/hdd/physio/vf2/rpeakfeats40df.csv', index_col = 0)\n",
    "dat = pd.read_csv('/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv', index_col = 0)\n",
    "dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "      <td>5519.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.406595</td>\n",
       "      <td>0.727104</td>\n",
       "      <td>0.112826</td>\n",
       "      <td>0.708937</td>\n",
       "      <td>104.548147</td>\n",
       "      <td>20.173163</td>\n",
       "      <td>75.118541</td>\n",
       "      <td>135.679331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.491243</td>\n",
       "      <td>0.306597</td>\n",
       "      <td>0.119441</td>\n",
       "      <td>0.305118</td>\n",
       "      <td>37.680188</td>\n",
       "      <td>17.949078</td>\n",
       "      <td>25.177756</td>\n",
       "      <td>60.399985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.300426</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>13.901761</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.901761</td>\n",
       "      <td>13.901761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.495020</td>\n",
       "      <td>0.019956</td>\n",
       "      <td>0.474800</td>\n",
       "      <td>67.607472</td>\n",
       "      <td>1.691655</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>73.891626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.597076</td>\n",
       "      <td>0.092168</td>\n",
       "      <td>0.585143</td>\n",
       "      <td>110.307537</td>\n",
       "      <td>19.709865</td>\n",
       "      <td>73.170732</td>\n",
       "      <td>147.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.909413</td>\n",
       "      <td>0.153371</td>\n",
       "      <td>0.902400</td>\n",
       "      <td>137.231137</td>\n",
       "      <td>35.309649</td>\n",
       "      <td>93.750000</td>\n",
       "      <td>189.873418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.316000</td>\n",
       "      <td>1.666000</td>\n",
       "      <td>4.316000</td>\n",
       "      <td>200.570512</td>\n",
       "      <td>83.200482</td>\n",
       "      <td>189.873418</td>\n",
       "      <td>238.095238</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                vf        RMSSD        STDNN      MEAN_RR      MEAN_HR  \\\n",
       "count  5519.000000  5519.000000  5519.000000  5519.000000  5519.000000   \n",
       "mean      0.406595     0.727104     0.112826     0.708937   104.548147   \n",
       "std       0.491243     0.306597     0.119441     0.305118    37.680188   \n",
       "min       0.000000     0.300426     0.000000     0.300000    13.901761   \n",
       "25%       0.000000     0.495020     0.019956     0.474800    67.607472   \n",
       "50%       0.000000     0.597076     0.092168     0.585143   110.307537   \n",
       "75%       1.000000     0.909413     0.153371     0.902400   137.231137   \n",
       "max       1.000000     4.316000     1.666000     4.316000   200.570512   \n",
       "\n",
       "            STD_HR       MIN_HR       MAX_HR  \n",
       "count  5519.000000  5519.000000  5519.000000  \n",
       "mean     20.173163    75.118541   135.679331  \n",
       "std      17.949078    25.177756    60.399985  \n",
       "min       0.000000    13.901761    13.901761  \n",
       "25%       1.691655    54.347826    73.891626  \n",
       "50%      19.709865    73.170732   147.058824  \n",
       "75%      35.309649    93.750000   189.873418  \n",
       "max      83.200482   189.873418   238.095238  "
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "      <td>3728.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.306599</td>\n",
       "      <td>0.754041</td>\n",
       "      <td>0.111176</td>\n",
       "      <td>0.732982</td>\n",
       "      <td>99.563097</td>\n",
       "      <td>16.078041</td>\n",
       "      <td>77.326587</td>\n",
       "      <td>122.761918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.461143</td>\n",
       "      <td>0.373724</td>\n",
       "      <td>0.159013</td>\n",
       "      <td>0.365165</td>\n",
       "      <td>32.055619</td>\n",
       "      <td>17.587056</td>\n",
       "      <td>27.741189</td>\n",
       "      <td>49.680256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.284000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.284000</td>\n",
       "      <td>12.355848</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.355848</td>\n",
       "      <td>12.355848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.545141</td>\n",
       "      <td>0.005844</td>\n",
       "      <td>0.534167</td>\n",
       "      <td>79.449510</td>\n",
       "      <td>0.571011</td>\n",
       "      <td>57.197486</td>\n",
       "      <td>83.682495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.684905</td>\n",
       "      <td>0.043153</td>\n",
       "      <td>0.656619</td>\n",
       "      <td>98.185194</td>\n",
       "      <td>7.161989</td>\n",
       "      <td>78.125000</td>\n",
       "      <td>112.781955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.819428</td>\n",
       "      <td>0.162685</td>\n",
       "      <td>0.778600</td>\n",
       "      <td>119.653589</td>\n",
       "      <td>29.500426</td>\n",
       "      <td>95.541401</td>\n",
       "      <td>168.539326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.856000</td>\n",
       "      <td>1.486000</td>\n",
       "      <td>4.856000</td>\n",
       "      <td>211.267606</td>\n",
       "      <td>100.579092</td>\n",
       "      <td>211.267606</td>\n",
       "      <td>238.095238</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                vf        RMSSD        STDNN      MEAN_RR      MEAN_HR  \\\n",
       "count  3728.000000  3728.000000  3728.000000  3728.000000  3728.000000   \n",
       "mean      0.306599     0.754041     0.111176     0.732982    99.563097   \n",
       "std       0.461143     0.373724     0.159013     0.365165    32.055619   \n",
       "min       0.000000     0.284000     0.000000     0.284000    12.355848   \n",
       "25%       0.000000     0.545141     0.005844     0.534167    79.449510   \n",
       "50%       0.000000     0.684905     0.043153     0.656619    98.185194   \n",
       "75%       1.000000     0.819428     0.162685     0.778600   119.653589   \n",
       "max       1.000000     4.856000     1.486000     4.856000   211.267606   \n",
       "\n",
       "            STD_HR       MIN_HR       MAX_HR  \n",
       "count  3728.000000  3728.000000  3728.000000  \n",
       "mean     16.078041    77.326587   122.761918  \n",
       "std      17.587056    27.741189    49.680256  \n",
       "min       0.000000    12.355848    12.355848  \n",
       "25%       0.571011    57.197486    83.682495  \n",
       "50%       7.161989    78.125000   112.781955  \n",
       "75%      29.500426    95.541401   168.539326  \n",
       "max     100.579092   211.267606   238.095238  "
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3275\n",
       "1    2244\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2585\n",
       "1    1143\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat = dat.drop(dat[dat['vf'].eq(0)].sample(750).index)\n",
    "dat.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, frame = train_test_split(frame, test_size=0.2, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = train_test_split(frame, test_size=0.2, random_state=42, shuffle=True)\n",
    "\n",
    "train = dat\n",
    "# train[train.columns[1:]] = scaler.fit_transform(train[train.columns[1:]])\n",
    "train = train.to_numpy()\n",
    "test = frame\n",
    "# test[test.columns[1:]] = scaler.transform(test[test.columns[1:]])\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try2vfcu\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    650\n",
       "1    454\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0    650\n",
       " 1    454\n",
       " Name: vf, dtype: int64, 0    2585\n",
       " 1    1143\n",
       " Name: vf, dtype: int64)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.vf.value_counts(), dat.vf.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <ins>Andha equal splitting vf and cu</ins>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    2585\n",
      "1    1143\n",
      "Name: vf, dtype: int64 \n",
      " 0    16275\n",
      "1     2244\n",
      "Name: vf, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(dat.vf.value_counts(), '\\n', frame.vf.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat0 = dat[dat.vf == 0][:1143].reset_index(drop = True)\n",
    "dat1 = dat[dat.vf == 1].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1138</th>\n",
       "      <td>0</td>\n",
       "      <td>1.418001</td>\n",
       "      <td>0.002000</td>\n",
       "      <td>1.418000</td>\n",
       "      <td>42.313201</td>\n",
       "      <td>0.059680</td>\n",
       "      <td>42.253521</td>\n",
       "      <td>42.372881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1139</th>\n",
       "      <td>0</td>\n",
       "      <td>1.422001</td>\n",
       "      <td>0.002000</td>\n",
       "      <td>1.422000</td>\n",
       "      <td>42.194176</td>\n",
       "      <td>0.059345</td>\n",
       "      <td>42.134831</td>\n",
       "      <td>42.253521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140</th>\n",
       "      <td>0</td>\n",
       "      <td>1.428004</td>\n",
       "      <td>0.003266</td>\n",
       "      <td>1.428000</td>\n",
       "      <td>42.017027</td>\n",
       "      <td>0.096098</td>\n",
       "      <td>41.899441</td>\n",
       "      <td>42.134831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141</th>\n",
       "      <td>0</td>\n",
       "      <td>1.434672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.434667</td>\n",
       "      <td>41.821850</td>\n",
       "      <td>0.109731</td>\n",
       "      <td>41.666667</td>\n",
       "      <td>41.899441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142</th>\n",
       "      <td>0</td>\n",
       "      <td>1.442001</td>\n",
       "      <td>0.002000</td>\n",
       "      <td>1.442000</td>\n",
       "      <td>41.608957</td>\n",
       "      <td>0.057710</td>\n",
       "      <td>41.551247</td>\n",
       "      <td>41.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1143 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD     STDNN   MEAN_RR    MEAN_HR    STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333  55.521205  1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000  57.253570  0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000  56.358432  1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000  56.305640  1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667  56.056575  0.964208  54.744526   \n",
       "...   ..       ...       ...       ...        ...       ...        ...   \n",
       "1138   0  1.418001  0.002000  1.418000  42.313201  0.059680  42.253521   \n",
       "1139   0  1.422001  0.002000  1.422000  42.194176  0.059345  42.134831   \n",
       "1140   0  1.428004  0.003266  1.428000  42.017027  0.096098  41.899441   \n",
       "1141   0  1.434672  0.003771  1.434667  41.821850  0.109731  41.666667   \n",
       "1142   0  1.442001  0.002000  1.442000  41.608957  0.057710  41.551247   \n",
       "\n",
       "         MAX_HR  \n",
       "0     57.471264  \n",
       "1     57.471264  \n",
       "2     57.251908  \n",
       "3     57.251908  \n",
       "4     57.034221  \n",
       "...         ...  \n",
       "1138  42.372881  \n",
       "1139  42.253521  \n",
       "1140  42.134831  \n",
       "1141  41.899441  \n",
       "1142  41.666667  \n",
       "\n",
       "[1143 rows x 8 columns]"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame0 = frame[frame.vf == 0][:1143].reset_index(drop = True)\n",
    "frame1 = frame[frame.vf == 1][:1143].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "\n",
    "def _train(df, split):\n",
    "    return df[df.index < (split * len(df))]\n",
    "\n",
    "# test\n",
    "\n",
    "def _test(df, split):\n",
    "    return df[df.index > (split * len(df))]\n",
    "\n",
    "train, test = list(), list()\n",
    "for d in [dat0, dat1]:\n",
    "    train.append(_train(d, 0.8))\n",
    "\n",
    "train = pd.concat(train, axis = 0).reset_index(drop = True)\n",
    "    \n",
    "for d in [frame0, frame1]:\n",
    "    test.append(_test(d, 0.8))\n",
    "    \n",
    "test = pd.concat(test, axis = 0).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    915\n",
       "1    915\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    228\n",
       "1    228\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.to_numpy()\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/tryeqsplitvfcu\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = train_test_split(dat, test_size = 0.2, shuffle = True)\n",
    "\n",
    "train = train.to_numpy()\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/try8\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2585\n",
       "1    1143\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "data0 = dat[dat.vf == 0][:1143].reset_index(drop = True)\n",
    "data1 = dat[dat.vf == 1].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "\n",
    "def _train(df, split):\n",
    "    return df[df.index < (split * len(df))]\n",
    "\n",
    "# test\n",
    "\n",
    "def _test(df, split):\n",
    "    return df[df.index > (split * len(df))]\n",
    "\n",
    "train, test = list(), list()\n",
    "for d in [data0, data1]:\n",
    "    train.append(_train(d, 0.8))\n",
    "\n",
    "train = pd.concat(train, axis = 0).reset_index(drop = True)\n",
    "    \n",
    "for d in [data0, data1]:\n",
    "    test.append(_test(d, 0.8))\n",
    "    \n",
    "test = pd.concat(test, axis = 0).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    915\n",
       "1    915\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    228\n",
       "1    228\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>SDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD      SDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR           lf            hf   total_power         vlf  \n",
       "0      57.471264  1939.478779   8359.852738  10740.573054  441.241538  \n",
       "1      57.471264  1712.922367   7112.985993   9249.068433  423.160073  \n",
       "2      57.251908  4196.568751  15537.499334  20534.333244  800.265159  \n",
       "3      57.251908  3692.942083  13527.269270  18097.231413  877.020061  \n",
       "4      57.034221  1816.833024   7521.108213   9787.482645  449.541409  \n",
       "...          ...          ...           ...           ...         ...  \n",
       "3723  164.835165  1056.532665   6478.046782   7743.907815  209.328367  \n",
       "3724   43.731778  1200.645300   3961.072822   5514.092053  352.373932  \n",
       "3725   43.859649  3101.967972  10454.393243  14430.419775  874.058561  \n",
       "3726   43.604651  3197.495564  11879.347598  15915.072325  838.229163  \n",
       "3727   43.352601  3363.977055  13430.154948  17599.920092  805.788089  \n",
       "\n",
       "[3728 rows x 12 columns]"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf = pd.read_csv('/hdd/physio/vf/finaldfs/CUDB_Bonsai_time_frequency_features.csv')\n",
    "newdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "_1 = newdf[newdf.vf == 1].reset_index(drop = True)\n",
    "_0 = newdf[newdf.vf == 0].reset_index(drop = True)[:1143]\n",
    "newdf = pd.concat([_1, _0], axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = train_test_split(newdf, test_size = 0.1, shuffle = True)\n",
    "\n",
    "train = pd.concat([_1[:1000], _0[:1000]])\n",
    "train = train.sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "\n",
    "test = pd.concat([_1[1000:], _0[1000:]])\n",
    "test = test.sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_CUDB_test_CUDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train, Test => AFDB, 2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Extraction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from numpy.lib.stride_tricks import as_strided as strided"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 2s, sys: 1min 1s, total: 3min 4s\n",
      "Wall time: 3min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "afdb = pd.read_csv('/hdd/physio/af/hugedfrpeak5sec.csv', index_col = 0).reset_index(drop = True)\n",
    "chdb = pd.read_csv('/hdd/physio/af2/finaldfs/2017final_normal_rpeak_L5_S1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((787154, 1251), (163593, 1251))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "afdb.shape, chdb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0    473522\n",
       " 1    313632\n",
       " Name: af, dtype: int64, 0    142189\n",
       " 1     21404\n",
       " Name: af, dtype: int64)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "afdb.af.value_counts(), chdb.af.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31min 58s, sys: 24.1 s, total: 32min 22s\n",
      "Wall time: 32min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def featureExtraction(df, fs):\n",
    "    df1 = df.af.copy()\n",
    "    df.drop(columns = ['af'], inplace = True)\n",
    "    final = list()\n",
    "    for i in range(len(df)):\n",
    "        peaks = []\n",
    "        peaks = df.loc[i][df.loc[i] == 1].index.values\n",
    "        peaks = [int(x) for x in peaks]\n",
    "        if (len(peaks) == 0 or len(peaks) == 1):\n",
    "            continue\n",
    "        af = df1.loc[i]\n",
    "        rr = np.diff(peaks) / fs \n",
    "        hr = 60 / rr\n",
    "        rmssd = np.sqrt(np.mean(np.square(rr)))\n",
    "        sdnn = np.std(rr)\n",
    "        mean_rr = np.mean(rr)\n",
    "        mean_hr = np.mean(hr)\n",
    "        std_hr = np.std(hr)\n",
    "        min_hr = np.min(hr)\n",
    "        max_hr = np.max(hr)\n",
    "        single_window_df = pd.DataFrame([[rmssd, sdnn, mean_rr, mean_hr, std_hr, min_hr, max_hr]], columns = ['RMSSD', 'STDNN', 'MEAN_RR', 'MEAN_HR', 'STD_HR', 'MIN_HR', 'MAX_HR'])\n",
    "        single_window_df.insert(0, 'af', af)\n",
    "        final.append(single_window_df)\n",
    "        del single_window_df\n",
    "    return pd.concat(final)\n",
    "\n",
    "afd = featureExtraction(afdb, 250)\n",
    "afd.reset_index(drop = True, inplace = True)\n",
    "chd = featureExtraction(chdb, 250)\n",
    "chd.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((769299, 8), (162628, 8))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "afd.shape, chd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "afd.to_csv('/hdd/physio/af/hugedf5secfeatures.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "chd.to_csv('/hdd/physio/af2/finaldfs/2017features.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train AFDB, Test 2017, Run Bonsai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 838 ms, sys: 85.5 ms, total: 924 ms\n",
      "Wall time: 1.72 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "afd = pd.read_csv('/hdd/physio/af/hugedf5secfeatures.csv')\n",
    "chd = pd.read_csv('/hdd/physio/af2/finaldfs/2017features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 69.6 ms, sys: 230 µs, total: 69.8 ms\n",
      "Wall time: 68.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_0 = afd[afd.af == 0].sample(n = 15000).reset_index(drop = True)\n",
    "_1 = afd[afd.af == 1].sample(n = 15000).reset_index(drop = True)\n",
    "train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "_0 = chd[chd.af == 0].sample(n = 1000).reset_index(drop = True)\n",
    "_1 = chd[chd.af == 1].sample(n = 1000).reset_index(drop = True)\n",
    "test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_AFDB_test_2017\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Train: 0, 1 = 15000, 15000\n",
    "Class 0:\n",
    "p, r, f, s = 0.90, 0.92, 0.91, 1000\n",
    "Class 1:\n",
    "p, r, f, s = 0.92, 0.90, 0.91, 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train 2017, Test 2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 313 ms, sys: 0 ns, total: 313 ms\n",
      "Wall time: 318 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "chd = pd.read_csv('/hdd/physio/af2/finaldfs/2017features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 41.3 ms, sys: 0 ns, total: 41.3 ms\n",
      "Wall time: 41.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_0 = chd[chd.af == 0].sample(n = 15000).reset_index(drop = True)\n",
    "_1 = chd[chd.af == 1].sample(n = 15000).reset_index(drop = True)\n",
    "train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "_0 = chd[chd.af == 0].sample(n = 1000).reset_index(drop = True)\n",
    "_1 = chd[chd.af == 1].sample(n = 1000).reset_index(drop = True)\n",
    "test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_2017_test_2017\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train 2017, Test AFDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.23 s, sys: 77.3 ms, total: 1.31 s\n",
      "Wall time: 1.65 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "chd = pd.read_csv('/hdd/physio/af/hugedf5secfeatures.csv')\n",
    "afd = pd.read_csv('/hdd/physio/af2/finaldfs/2017features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 117 ms, sys: 0 ns, total: 117 ms\n",
      "Wall time: 290 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_0 = afd[afd.af == 0].sample(n = 15000).reset_index(drop = True)\n",
    "_1 = afd[afd.af == 1].sample(n = 15000).reset_index(drop = True)\n",
    "train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "_0 = chd[chd.af == 0].sample(n = 1000).reset_index(drop = True)\n",
    "_1 = chd[chd.af == 1].sample(n = 1000).reset_index(drop = True)\n",
    "test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_2017_test_AFDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train CUDB, Test VFDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vfdb = pd.read_csv('/hdd/physio/vf2/rpeakfeats40df.csv', index_col = 0)\n",
    "cudb = pd.read_csv('/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3275\n",
       "1    2244\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vfdb.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2585\n",
       "1    1143\n",
       "Name: vf, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cudb.vf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.19 ms, sys: 546 µs, total: 1.73 ms\n",
      "Wall time: 1.02 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# _0 = cudb[cudb.vf == 0].sample(n = 1143).reset_index(drop = True)\n",
    "# _1 = cudb[cudb.vf == 1].sample(n = 1143).reset_index(drop = True)\n",
    "# train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "# train = train.to_numpy()\n",
    "# _0 = vfdb[vfdb.vf == 0].sample(n = 500).reset_index(drop = True)\n",
    "# _1 = vfdb[vfdb.vf == 1].sample(n = 500).reset_index(drop = True)\n",
    "# test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "# test = test.to_numpy()\n",
    "\n",
    "train = cudb.to_numpy()\n",
    "test = vfdb.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_CUDB_test_VFDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train AFDB, Test AFDB (5 second length, 1 second stride)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 717 ms, sys: 24.6 ms, total: 742 ms\n",
      "Wall time: 740 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "afd = pd.read_csv('/hdd/physio/af/hugedf5secfeatures.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 386 ms, sys: 15.5 ms, total: 402 ms\n",
      "Wall time: 401 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_0 = afd[afd.af == 0].sample(n = 18000).reset_index(drop = True)\n",
    "_1 = afd[afd.af == 1].sample(n = 18000).reset_index(drop = True)\n",
    "data = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "data.to_csv('/hdd/physio/af/finalscore.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.read_csv('/hdd/physio/af/finalscore.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    18000\n",
       "1    18000\n",
       "Name: af, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.af.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 166 ms, sys: 0 ns, total: 166 ms\n",
      "Wall time: 201 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_0 = afd[afd.af == 0].sample(n = 15000).reset_index(drop = True)\n",
    "_1 = afd[afd.af == 1].sample(n = 15000).reset_index(drop = True)\n",
    "train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "_0 = afd[afd.af == 0].sample(n = 1000).reset_index(drop = True)\n",
    "_1 = afd[afd.af == 1].sample(n = 1000).reset_index(drop = True)\n",
    "test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_AFDB_test_AFDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train VFDB, Test VFDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.6 ms, sys: 962 µs, total: 13.6 ms\n",
      "Wall time: 13.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vfdb = pd.read_csv('/hdd/physio/vf2/rpeakfeats40df.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>STDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.811915</td>\n",
       "      <td>0.129011</td>\n",
       "      <td>0.801600</td>\n",
       "      <td>77.221380</td>\n",
       "      <td>14.874714</td>\n",
       "      <td>63.559322</td>\n",
       "      <td>105.633803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.821033</td>\n",
       "      <td>0.083269</td>\n",
       "      <td>0.816800</td>\n",
       "      <td>74.283999</td>\n",
       "      <td>8.159667</td>\n",
       "      <td>64.377682</td>\n",
       "      <td>89.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.783983</td>\n",
       "      <td>0.091146</td>\n",
       "      <td>0.778667</td>\n",
       "      <td>78.406371</td>\n",
       "      <td>11.552355</td>\n",
       "      <td>71.770335</td>\n",
       "      <td>104.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.832527</td>\n",
       "      <td>0.139241</td>\n",
       "      <td>0.820800</td>\n",
       "      <td>75.626624</td>\n",
       "      <td>15.101854</td>\n",
       "      <td>60.240964</td>\n",
       "      <td>104.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.824388</td>\n",
       "      <td>0.155447</td>\n",
       "      <td>0.809600</td>\n",
       "      <td>77.066327</td>\n",
       "      <td>15.484618</td>\n",
       "      <td>60.728745</td>\n",
       "      <td>99.337748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5514</th>\n",
       "      <td>0</td>\n",
       "      <td>0.612400</td>\n",
       "      <td>0.222629</td>\n",
       "      <td>0.570500</td>\n",
       "      <td>124.065656</td>\n",
       "      <td>48.438161</td>\n",
       "      <td>74.257426</td>\n",
       "      <td>176.470588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5515</th>\n",
       "      <td>0</td>\n",
       "      <td>0.582319</td>\n",
       "      <td>0.222134</td>\n",
       "      <td>0.538286</td>\n",
       "      <td>131.458630</td>\n",
       "      <td>48.458291</td>\n",
       "      <td>74.257426</td>\n",
       "      <td>174.418605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5516</th>\n",
       "      <td>0</td>\n",
       "      <td>0.615648</td>\n",
       "      <td>0.228932</td>\n",
       "      <td>0.571500</td>\n",
       "      <td>124.999664</td>\n",
       "      <td>49.997374</td>\n",
       "      <td>72.463768</td>\n",
       "      <td>178.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5517</th>\n",
       "      <td>0</td>\n",
       "      <td>0.575841</td>\n",
       "      <td>0.219000</td>\n",
       "      <td>0.532571</td>\n",
       "      <td>132.753298</td>\n",
       "      <td>48.892212</td>\n",
       "      <td>75.757576</td>\n",
       "      <td>180.722892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5518</th>\n",
       "      <td>0</td>\n",
       "      <td>0.607269</td>\n",
       "      <td>0.220045</td>\n",
       "      <td>0.566000</td>\n",
       "      <td>124.888866</td>\n",
       "      <td>48.577367</td>\n",
       "      <td>75.757576</td>\n",
       "      <td>176.470588</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5519 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD     STDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  0.811915  0.129011  0.801600   77.221380  14.874714  63.559322   \n",
       "1      0  0.821033  0.083269  0.816800   74.283999   8.159667  64.377682   \n",
       "2      0  0.783983  0.091146  0.778667   78.406371  11.552355  71.770335   \n",
       "3      0  0.832527  0.139241  0.820800   75.626624  15.101854  60.240964   \n",
       "4      0  0.824388  0.155447  0.809600   77.066327  15.484618  60.728745   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "5514   0  0.612400  0.222629  0.570500  124.065656  48.438161  74.257426   \n",
       "5515   0  0.582319  0.222134  0.538286  131.458630  48.458291  74.257426   \n",
       "5516   0  0.615648  0.228932  0.571500  124.999664  49.997374  72.463768   \n",
       "5517   0  0.575841  0.219000  0.532571  132.753298  48.892212  75.757576   \n",
       "5518   0  0.607269  0.220045  0.566000  124.888866  48.577367  75.757576   \n",
       "\n",
       "          MAX_HR  \n",
       "0     105.633803  \n",
       "1      89.285714  \n",
       "2     104.166667  \n",
       "3     104.166667  \n",
       "4      99.337748  \n",
       "...          ...  \n",
       "5514  176.470588  \n",
       "5515  174.418605  \n",
       "5516  178.571429  \n",
       "5517  180.722892  \n",
       "5518  176.470588  \n",
       "\n",
       "[5519 rows x 8 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vfdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.86 ms, sys: 0 ns, total: 4.86 ms\n",
      "Wall time: 4.38 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# _0 = afd[afd.af == 0].sample(n = 15000).reset_index(drop = True)\n",
    "# _1 = afd[afd.af == 1].sample(n = 15000).reset_index(drop = True)\n",
    "# train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "# train = train.to_numpy()\n",
    "# _0 = afd[afd.af == 0].sample(n = 1000).reset_index(drop = True)\n",
    "# _1 = afd[afd.af == 1].sample(n = 1000).reset_index(drop = True)\n",
    "# test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "# test = test.to_numpy()\n",
    "train, test = train_test_split(vfdb, test_size = 0.2, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_VFDB_test_VFDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train.to_numpy())\n",
    "np.save(path + '/test.npy', test.to_numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train CUDB, Test CUDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.5 ms, sys: 0 ns, total: 14.5 ms\n",
      "Wall time: 13.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cudb = pd.read_csv('/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    2325\n",
      "1    1030\n",
      "Name: vf, dtype: int64 0    260\n",
      "1    113\n",
      "Name: vf, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "train, test = train_test_split(cudb, test_size = 0.1, random_state = 42)\n",
    "print(train.vf.value_counts(), test.vf.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_CUDB_test_CUDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train.to_numpy())\n",
    "np.save(path + '/test.npy', test.to_numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train CUDB, Test VFDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 22.1 ms, sys: 0 ns, total: 22.1 ms\n",
      "Wall time: 21.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vfdb = pd.read_csv('/hdd/physio/vf2/rpeakfeats40df.csv', index_col = 0)\n",
    "cudb = pd.read_csv('/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "_0 = cudb[cudb.vf == 0].sample(n = 1143).reset_index(drop = True)\n",
    "_1 = cudb[cudb.vf == 1].sample(n = 1143).reset_index(drop = True)\n",
    "train = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "train = train.to_numpy()\n",
    "_0 = vfdb[vfdb.vf == 0].sample(n = 2244).reset_index(drop = True)\n",
    "_1 = vfdb[vfdb.vf == 1].sample(n = 2244).reset_index(drop = True)\n",
    "test = pd.concat([_0, _1], axis = 0).sample(frac = 1).reset_index(drop = True)\n",
    "test = test.to_numpy()\n",
    "\n",
    "# train = cudb.to_numpy()\n",
    "# test = vfdb.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_CUDB_test_VFDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0    2585\n",
       " 1    1143\n",
       " Name: vf, dtype: int64, 0    3275\n",
       " 1    2244\n",
       " Name: vf, dtype: int64)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cudb.vf.value_counts(), vfdb.vf.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train VFDB, Test CUDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24.8 ms, sys: 0 ns, total: 24.8 ms\n",
      "Wall time: 24.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vfdb = pd.read_csv('/hdd/physio/vf2/rpeakfeats40df.csv', index_col = 0)\n",
    "cudb = pd.read_csv('/hdd/physio/vf/finaldfs/trybonsaifeatures3.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = vfdb.to_numpy()\n",
    "test = cudb.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/train_VFDB_test_CUDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train)\n",
    "np.save(path + '/test.npy', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying time+freq domain features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train CUDB, Test CUDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>SDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>lf_hf_ratio</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>0.231999</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>0.240816</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>0.270093</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>0.273000</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>0.241565</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>0.163094</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>0.303111</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>0.296714</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>0.269164</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>0.250479</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3716 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD      SDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR           lf            hf  lf_hf_ratio   total_power  \\\n",
       "0      57.471264  1939.478779   8359.852738     0.231999  10740.573054   \n",
       "1      57.471264  1712.922367   7112.985993     0.240816   9249.068433   \n",
       "2      57.251908  4196.568751  15537.499334     0.270093  20534.333244   \n",
       "3      57.251908  3692.942083  13527.269270     0.273000  18097.231413   \n",
       "4      57.034221  1816.833024   7521.108213     0.241565   9787.482645   \n",
       "...          ...          ...           ...          ...           ...   \n",
       "3723  164.835165  1056.532665   6478.046782     0.163094   7743.907815   \n",
       "3724   43.731778  1200.645300   3961.072822     0.303111   5514.092053   \n",
       "3725   43.859649  3101.967972  10454.393243     0.296714  14430.419775   \n",
       "3726   43.604651  3197.495564  11879.347598     0.269164  15915.072325   \n",
       "3727   43.352601  3363.977055  13430.154948     0.250479  17599.920092   \n",
       "\n",
       "             vlf  \n",
       "0     441.241538  \n",
       "1     423.160073  \n",
       "2     800.265159  \n",
       "3     877.020061  \n",
       "4     449.541409  \n",
       "...          ...  \n",
       "3723  209.328367  \n",
       "3724  352.373932  \n",
       "3725  874.058561  \n",
       "3726  838.229163  \n",
       "3727  805.788089  \n",
       "\n",
       "[3716 rows x 13 columns]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cu = pd.read_csv('/hdd/physio/vf/finaldfs/time_freq_features.csv', index_col = 0)\n",
    "cu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    2326\n",
      "1    1018\n",
      "Name: vf, dtype: int64 0    257\n",
      "1    115\n",
      "Name: vf, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "train, test = train_test_split(cu, test_size = 0.1, random_state = 42)\n",
    "print(train.vf.value_counts(), test.vf.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/freq_train_CUDB_test_CUDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train.to_numpy())\n",
    "np.save(path + '/test.npy', test.to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>SDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf     RMSSD      SDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.081660  0.026600  1.081333   55.521205   1.388378  54.347826   \n",
       "1      0  1.048015  0.005657  1.048000   57.253570   0.307866  56.818182   \n",
       "2      0  1.065197  0.020469  1.065000   56.358432   1.061594  54.545455   \n",
       "3      0  1.066197  0.020494  1.066000   56.305640   1.064400  54.545455   \n",
       "4      0  1.070828  0.018571  1.070667   56.056575   0.964208  54.744526   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "3723   1  0.887601  0.410447  0.787000  101.853431  50.658017  43.988270   \n",
       "3724   1  1.382036  0.010000  1.382000   43.417613   0.314165  43.103448   \n",
       "3725   1  1.378703  0.009978  1.378667   43.522583   0.314125  43.103448   \n",
       "3726   1  1.378672  0.003771  1.378667   43.520634   0.118818  43.352601   \n",
       "3727   1  1.393358  0.008219  1.393333   43.063697   0.253679  42.735043   \n",
       "\n",
       "          MAX_HR           lf            hf   total_power         vlf  \n",
       "0      57.471264  1939.478779   8359.852738  10740.573054  441.241538  \n",
       "1      57.471264  1712.922367   7112.985993   9249.068433  423.160073  \n",
       "2      57.251908  4196.568751  15537.499334  20534.333244  800.265159  \n",
       "3      57.251908  3692.942083  13527.269270  18097.231413  877.020061  \n",
       "4      57.034221  1816.833024   7521.108213   9787.482645  449.541409  \n",
       "...          ...          ...           ...           ...         ...  \n",
       "3723  164.835165  1056.532665   6478.046782   7743.907815  209.328367  \n",
       "3724   43.731778  1200.645300   3961.072822   5514.092053  352.373932  \n",
       "3725   43.859649  3101.967972  10454.393243  14430.419775  874.058561  \n",
       "3726   43.604651  3197.495564  11879.347598  15915.072325  838.229163  \n",
       "3727   43.352601  3363.977055  13430.154948  17599.920092  805.788089  \n",
       "\n",
       "[3728 rows x 12 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('/hdd/physio/vf/finaldfs/time_freq_features.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>SDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.081660</td>\n",
       "      <td>0.026600</td>\n",
       "      <td>1.081333</td>\n",
       "      <td>55.521205</td>\n",
       "      <td>1.388378</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1939.478779</td>\n",
       "      <td>8359.852738</td>\n",
       "      <td>10740.573054</td>\n",
       "      <td>441.241538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.048015</td>\n",
       "      <td>0.005657</td>\n",
       "      <td>1.048000</td>\n",
       "      <td>57.253570</td>\n",
       "      <td>0.307866</td>\n",
       "      <td>56.818182</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>1712.922367</td>\n",
       "      <td>7112.985993</td>\n",
       "      <td>9249.068433</td>\n",
       "      <td>423.160073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.065197</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>1.065000</td>\n",
       "      <td>56.358432</td>\n",
       "      <td>1.061594</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>4196.568751</td>\n",
       "      <td>15537.499334</td>\n",
       "      <td>20534.333244</td>\n",
       "      <td>800.265159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1.066197</td>\n",
       "      <td>0.020494</td>\n",
       "      <td>1.066000</td>\n",
       "      <td>56.305640</td>\n",
       "      <td>1.064400</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>57.251908</td>\n",
       "      <td>3692.942083</td>\n",
       "      <td>13527.269270</td>\n",
       "      <td>18097.231413</td>\n",
       "      <td>877.020061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1.070828</td>\n",
       "      <td>0.018571</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.056575</td>\n",
       "      <td>0.964208</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>57.034221</td>\n",
       "      <td>1816.833024</td>\n",
       "      <td>7521.108213</td>\n",
       "      <td>9787.482645</td>\n",
       "      <td>449.541409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>3723</td>\n",
       "      <td>1</td>\n",
       "      <td>0.887601</td>\n",
       "      <td>0.410447</td>\n",
       "      <td>0.787000</td>\n",
       "      <td>101.853431</td>\n",
       "      <td>50.658017</td>\n",
       "      <td>43.988270</td>\n",
       "      <td>164.835165</td>\n",
       "      <td>1056.532665</td>\n",
       "      <td>6478.046782</td>\n",
       "      <td>7743.907815</td>\n",
       "      <td>209.328367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3724</th>\n",
       "      <td>3724</td>\n",
       "      <td>1</td>\n",
       "      <td>1.382036</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>1.382000</td>\n",
       "      <td>43.417613</td>\n",
       "      <td>0.314165</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.731778</td>\n",
       "      <td>1200.645300</td>\n",
       "      <td>3961.072822</td>\n",
       "      <td>5514.092053</td>\n",
       "      <td>352.373932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3725</th>\n",
       "      <td>3725</td>\n",
       "      <td>1</td>\n",
       "      <td>1.378703</td>\n",
       "      <td>0.009978</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.522583</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>43.859649</td>\n",
       "      <td>3101.967972</td>\n",
       "      <td>10454.393243</td>\n",
       "      <td>14430.419775</td>\n",
       "      <td>874.058561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3726</th>\n",
       "      <td>3726</td>\n",
       "      <td>1</td>\n",
       "      <td>1.378672</td>\n",
       "      <td>0.003771</td>\n",
       "      <td>1.378667</td>\n",
       "      <td>43.520634</td>\n",
       "      <td>0.118818</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>43.604651</td>\n",
       "      <td>3197.495564</td>\n",
       "      <td>11879.347598</td>\n",
       "      <td>15915.072325</td>\n",
       "      <td>838.229163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td>3727</td>\n",
       "      <td>1</td>\n",
       "      <td>1.393358</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>1.393333</td>\n",
       "      <td>43.063697</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>42.735043</td>\n",
       "      <td>43.352601</td>\n",
       "      <td>3363.977055</td>\n",
       "      <td>13430.154948</td>\n",
       "      <td>17599.920092</td>\n",
       "      <td>805.788089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3728 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  vf     RMSSD      SDNN   MEAN_RR     MEAN_HR     STD_HR  \\\n",
       "0              0   0  1.081660  0.026600  1.081333   55.521205   1.388378   \n",
       "1              1   0  1.048015  0.005657  1.048000   57.253570   0.307866   \n",
       "2              2   0  1.065197  0.020469  1.065000   56.358432   1.061594   \n",
       "3              3   0  1.066197  0.020494  1.066000   56.305640   1.064400   \n",
       "4              4   0  1.070828  0.018571  1.070667   56.056575   0.964208   \n",
       "...          ...  ..       ...       ...       ...         ...        ...   \n",
       "3723        3723   1  0.887601  0.410447  0.787000  101.853431  50.658017   \n",
       "3724        3724   1  1.382036  0.010000  1.382000   43.417613   0.314165   \n",
       "3725        3725   1  1.378703  0.009978  1.378667   43.522583   0.314125   \n",
       "3726        3726   1  1.378672  0.003771  1.378667   43.520634   0.118818   \n",
       "3727        3727   1  1.393358  0.008219  1.393333   43.063697   0.253679   \n",
       "\n",
       "         MIN_HR      MAX_HR           lf            hf   total_power  \\\n",
       "0     54.347826   57.471264  1939.478779   8359.852738  10740.573054   \n",
       "1     56.818182   57.471264  1712.922367   7112.985993   9249.068433   \n",
       "2     54.545455   57.251908  4196.568751  15537.499334  20534.333244   \n",
       "3     54.545455   57.251908  3692.942083  13527.269270  18097.231413   \n",
       "4     54.744526   57.034221  1816.833024   7521.108213   9787.482645   \n",
       "...         ...         ...          ...           ...           ...   \n",
       "3723  43.988270  164.835165  1056.532665   6478.046782   7743.907815   \n",
       "3724  43.103448   43.731778  1200.645300   3961.072822   5514.092053   \n",
       "3725  43.103448   43.859649  3101.967972  10454.393243  14430.419775   \n",
       "3726  43.352601   43.604651  3197.495564  11879.347598  15915.072325   \n",
       "3727  42.735043   43.352601  3363.977055  13430.154948  17599.920092   \n",
       "\n",
       "             vlf  \n",
       "0     441.241538  \n",
       "1     423.160073  \n",
       "2     800.265159  \n",
       "3     877.020061  \n",
       "4     449.541409  \n",
       "...          ...  \n",
       "3723  209.328367  \n",
       "3724  352.373932  \n",
       "3725  874.058561  \n",
       "3726  838.229163  \n",
       "3727  805.788089  \n",
       "\n",
       "[3728 rows x 13 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('/hdd/physio/vf2/time_freq_features.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All detectors rpeak databases feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from ecgdetectors import Detectors\n",
    "from time import time as time\n",
    "import seaborn as sns\n",
    "from hrvanalysis import remove_outliers, remove_ectopic_beats, interpolate_nan_values, get_time_domain_features, get_geometrical_features, get_frequency_domain_features, get_csi_cvi_features, get_poincare_plot_features, get_sampen\n",
    "from time import time as time\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/hdd/physio/vf/finaldfs/'\n",
    "fs = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>1240</th>\n",
       "      <th>1241</th>\n",
       "      <th>1242</th>\n",
       "      <th>1243</th>\n",
       "      <th>1244</th>\n",
       "      <th>1245</th>\n",
       "      <th>1246</th>\n",
       "      <th>1247</th>\n",
       "      <th>1248</th>\n",
       "      <th>1249</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5399</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5400</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5401</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5402</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5403</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5404 rows × 1251 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      vf  0  1  2  3  4  5  6  7  8  ...  1240  1241  1242  1243  1244  1245  \\\n",
       "0      0  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "1      0  0  0  0  0  0  0  0  1  0  ...     0     0     0     0     0     0   \n",
       "2      0  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     1     0   \n",
       "3      0  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "4      0  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "...   .. .. .. .. .. .. .. .. .. ..  ...   ...   ...   ...   ...   ...   ...   \n",
       "5399   1  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "5400   1  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "5401   1  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "5402   1  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "5403   1  0  0  0  0  0  0  0  0  0  ...     0     0     0     0     0     0   \n",
       "\n",
       "      1246  1247  1248  1249  \n",
       "0        0     0     0     0  \n",
       "1        0     0     0     0  \n",
       "2        0     0     0     0  \n",
       "3        0     0     0     0  \n",
       "4        0     0     0     0  \n",
       "...    ...   ...   ...   ...  \n",
       "5399     0     0     0     0  \n",
       "5400     0     0     0     0  \n",
       "5401     0     0     0     0  \n",
       "5402     0     0     0     0  \n",
       "5403     0     0     0     0  \n",
       "\n",
       "[5404 rows x 1251 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path + 'finalrpeak5secpt.csv', index_col = 0)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20 s, sys: 7.67 ms, total: 20 s\n",
      "Wall time: 20 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def featureExtraction(df, fs):\n",
    "    df1 = df.vf.copy()\n",
    "    df.drop(columns = ['vf'], inplace = True)\n",
    "    final = list()\n",
    "    for i in range(len(df)):\n",
    "        peaks = []\n",
    "        peaks = df.loc[i][df.loc[i] == 1].index.values\n",
    "        peaks = [int(x) for x in peaks]\n",
    "        if (len(peaks) == 0 or len(peaks) == 1):\n",
    "            continue\n",
    "        vf = df1.loc[i]\n",
    "        rr = np.diff(peaks) / fs \n",
    "        hr = np.diff(peaks) / fs \n",
    "        hr = 60 / rr\n",
    "        rmssd = np.sqrt(np.mean(np.square(rr)))\n",
    "        sdnn = np.std(rr)\n",
    "        mean_rr = np.mean(rr)\n",
    "        mean_hr = np.mean(hr)\n",
    "        std_hr = np.std(hr)\n",
    "        min_hr = np.min(hr)\n",
    "        max_hr = np.max(hr)\n",
    "        time_domain_features = {'RMSSD': rmssd, 'SDNN': sdnn, 'MEAN_RR': mean_rr, 'MEAN_HR': mean_hr, 'STD_HR': std_hr, 'MIN_HR': min_hr, 'MAX_HR': max_hr}\n",
    "        frequency_domain_features = get_frequency_domain_features(peaks)\n",
    "        features = [time_domain_features, frequency_domain_features]\n",
    "        del peaks\n",
    "        new = {}\n",
    "        for x in features:\n",
    "            for y in x:\n",
    "                new[y] = x[y]\n",
    "\n",
    "        single_window_df = pd.DataFrame(list(new.items()))\n",
    "        single_window_df = single_window_df.transpose()\n",
    "        columns = single_window_df.iloc[0]\n",
    "        single_window_df = single_window_df[1:]\n",
    "        single_window_df.columns = columns\n",
    "        single_window_df.drop(columns = ['lfnu', 'hfnu'], inplace = True)\n",
    "        single_window_df.insert(0, 'vf', vf)\n",
    "        final.append(single_window_df)\n",
    "        del single_window_df\n",
    "    return pd.concat(final)\n",
    "\n",
    "finaldf = featureExtraction(df, fs)\n",
    "finaldf.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vf</th>\n",
       "      <th>RMSSD</th>\n",
       "      <th>SDNN</th>\n",
       "      <th>MEAN_RR</th>\n",
       "      <th>MEAN_HR</th>\n",
       "      <th>STD_HR</th>\n",
       "      <th>MIN_HR</th>\n",
       "      <th>MAX_HR</th>\n",
       "      <th>lf</th>\n",
       "      <th>hf</th>\n",
       "      <th>lf_hf_ratio</th>\n",
       "      <th>total_power</th>\n",
       "      <th>vlf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.067852</td>\n",
       "      <td>0.04265</td>\n",
       "      <td>1.067</td>\n",
       "      <td>56.325866</td>\n",
       "      <td>2.338151</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>60.240964</td>\n",
       "      <td>4258.342621</td>\n",
       "      <td>15225.922189</td>\n",
       "      <td>0.279677</td>\n",
       "      <td>20435.091602</td>\n",
       "      <td>950.826792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.049102</td>\n",
       "      <td>0.048083</td>\n",
       "      <td>1.048</td>\n",
       "      <td>57.372743</td>\n",
       "      <td>2.633691</td>\n",
       "      <td>54.744526</td>\n",
       "      <td>60.240964</td>\n",
       "      <td>3384.789616</td>\n",
       "      <td>12164.965843</td>\n",
       "      <td>0.278241</td>\n",
       "      <td>16415.910198</td>\n",
       "      <td>866.154739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.065779</td>\n",
       "      <td>0.040731</td>\n",
       "      <td>1.065</td>\n",
       "      <td>56.422114</td>\n",
       "      <td>2.199265</td>\n",
       "      <td>54.347826</td>\n",
       "      <td>59.760956</td>\n",
       "      <td>4197.698864</td>\n",
       "      <td>15539.679612</td>\n",
       "      <td>0.270128</td>\n",
       "      <td>20536.859778</td>\n",
       "      <td>799.481303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.054609</td>\n",
       "      <td>0.035833</td>\n",
       "      <td>1.054</td>\n",
       "      <td>56.992663</td>\n",
       "      <td>1.961929</td>\n",
       "      <td>54.545455</td>\n",
       "      <td>60.0</td>\n",
       "      <td>3634.194243</td>\n",
       "      <td>13257.552165</td>\n",
       "      <td>0.274123</td>\n",
       "      <td>17757.79605</td>\n",
       "      <td>866.049642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.071833</td>\n",
       "      <td>0.049996</td>\n",
       "      <td>1.070667</td>\n",
       "      <td>56.166484</td>\n",
       "      <td>2.711889</td>\n",
       "      <td>54.151625</td>\n",
       "      <td>60.0</td>\n",
       "      <td>1788.017035</td>\n",
       "      <td>7615.317551</td>\n",
       "      <td>0.234792</td>\n",
       "      <td>9838.797681</td>\n",
       "      <td>435.463095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4159</th>\n",
       "      <td>1</td>\n",
       "      <td>0.618066</td>\n",
       "      <td>0.305415</td>\n",
       "      <td>0.537333</td>\n",
       "      <td>144.83664</td>\n",
       "      <td>58.683088</td>\n",
       "      <td>58.365759</td>\n",
       "      <td>194.805195</td>\n",
       "      <td>3465.097837</td>\n",
       "      <td>12196.015181</td>\n",
       "      <td>0.284117</td>\n",
       "      <td>16120.01757</td>\n",
       "      <td>458.904552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4160</th>\n",
       "      <td>1</td>\n",
       "      <td>0.767839</td>\n",
       "      <td>0.347574</td>\n",
       "      <td>0.684667</td>\n",
       "      <td>118.201576</td>\n",
       "      <td>60.34155</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>185.185185</td>\n",
       "      <td>5032.684618</td>\n",
       "      <td>13303.941056</td>\n",
       "      <td>0.378285</td>\n",
       "      <td>19182.002359</td>\n",
       "      <td>845.376686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4161</th>\n",
       "      <td>1</td>\n",
       "      <td>0.722874</td>\n",
       "      <td>0.324961</td>\n",
       "      <td>0.645714</td>\n",
       "      <td>119.585478</td>\n",
       "      <td>53.302582</td>\n",
       "      <td>57.471264</td>\n",
       "      <td>182.926829</td>\n",
       "      <td>6824.984046</td>\n",
       "      <td>14029.554394</td>\n",
       "      <td>0.486472</td>\n",
       "      <td>21892.743508</td>\n",
       "      <td>1038.205069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4162</th>\n",
       "      <td>1</td>\n",
       "      <td>0.727125</td>\n",
       "      <td>0.346182</td>\n",
       "      <td>0.639429</td>\n",
       "      <td>127.552448</td>\n",
       "      <td>64.228388</td>\n",
       "      <td>56.179775</td>\n",
       "      <td>227.272727</td>\n",
       "      <td>7966.202998</td>\n",
       "      <td>12214.947378</td>\n",
       "      <td>0.652168</td>\n",
       "      <td>21272.954668</td>\n",
       "      <td>1091.804292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4163</th>\n",
       "      <td>1</td>\n",
       "      <td>0.775581</td>\n",
       "      <td>0.360575</td>\n",
       "      <td>0.686667</td>\n",
       "      <td>122.476039</td>\n",
       "      <td>68.450687</td>\n",
       "      <td>56.603774</td>\n",
       "      <td>227.272727</td>\n",
       "      <td>5127.912486</td>\n",
       "      <td>13625.637734</td>\n",
       "      <td>0.376343</td>\n",
       "      <td>19474.638545</td>\n",
       "      <td>721.088324</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4153 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0     vf     RMSSD      SDNN   MEAN_RR     MEAN_HR     STD_HR     MIN_HR  \\\n",
       "0      0  1.067852   0.04265     1.067   56.325866   2.338151  54.347826   \n",
       "1      0  1.049102  0.048083     1.048   57.372743   2.633691  54.744526   \n",
       "2      0  1.065779  0.040731     1.065   56.422114   2.199265  54.347826   \n",
       "3      0  1.054609  0.035833     1.054   56.992663   1.961929  54.545455   \n",
       "4      0  1.071833  0.049996  1.070667   56.166484   2.711889  54.151625   \n",
       "...   ..       ...       ...       ...         ...        ...        ...   \n",
       "4159   1  0.618066  0.305415  0.537333   144.83664  58.683088  58.365759   \n",
       "4160   1  0.767839  0.347574  0.684667  118.201576   60.34155  57.471264   \n",
       "4161   1  0.722874  0.324961  0.645714  119.585478  53.302582  57.471264   \n",
       "4162   1  0.727125  0.346182  0.639429  127.552448  64.228388  56.179775   \n",
       "4163   1  0.775581  0.360575  0.686667  122.476039  68.450687  56.603774   \n",
       "\n",
       "0         MAX_HR           lf            hf lf_hf_ratio   total_power  \\\n",
       "0      60.240964  4258.342621  15225.922189    0.279677  20435.091602   \n",
       "1      60.240964  3384.789616  12164.965843    0.278241  16415.910198   \n",
       "2      59.760956  4197.698864  15539.679612    0.270128  20536.859778   \n",
       "3           60.0  3634.194243  13257.552165    0.274123   17757.79605   \n",
       "4           60.0  1788.017035   7615.317551    0.234792   9838.797681   \n",
       "...          ...          ...           ...         ...           ...   \n",
       "4159  194.805195  3465.097837  12196.015181    0.284117   16120.01757   \n",
       "4160  185.185185  5032.684618  13303.941056    0.378285  19182.002359   \n",
       "4161  182.926829  6824.984046  14029.554394    0.486472  21892.743508   \n",
       "4162  227.272727  7966.202998  12214.947378    0.652168  21272.954668   \n",
       "4163  227.272727  5127.912486  13625.637734    0.376343  19474.638545   \n",
       "\n",
       "0             vlf  \n",
       "0      950.826792  \n",
       "1      866.154739  \n",
       "2      799.481303  \n",
       "3      866.049642  \n",
       "4      435.463095  \n",
       "...           ...  \n",
       "4159   458.904552  \n",
       "4160   845.376686  \n",
       "4161  1038.205069  \n",
       "4162  1091.804292  \n",
       "4163   721.088324  \n",
       "\n",
       "[4153 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finaldf = finaldf.dropna(how='any')\n",
    "finaldf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "finaldf.to_csv('/hdd/physio/vf/pttimefreqfeatures.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use all detectors to train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "hs = pd.read_csv('/hdd/physio/vf/hstimefreqfeatures.csv')\n",
    "pt = pd.read_csv('/hdd/physio/vf/pttimefreqfeatures.csv')\n",
    "sswt = pd.read_csv('/hdd/physio/vf/sswttimefreqfeatures.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(sswt, test_size = 0.1, random_state = 42)\n",
    "\n",
    "path = \"/hdd/physio/edgeml/examples/pytorch/Bonsai/sswt_freq_train_CUDB_test_CUDB\"\n",
    "\n",
    "np.save(path + '/train.npy', train.to_numpy())\n",
    "np.save(path + '/test.npy', test.to_numpy())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:physio]",
   "language": "python",
   "name": "conda-env-physio-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}