label_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "import numpy as np\n",
    "\n",
    "GT_DIR = '/path/to/your/input/data/directory/'\n",
    "GEN_DIR = '/path/to/your/output/data/directory/'\n",
    "\n",
    "\n",
    "CXR_LABELS_1 = [\"Enlarged Cardiomediastinum\", \"Cardiomegaly\", \"Lung Lesion\", \n",
    "\"Lung Opacity\", \"Edema\", \"Consolidation\", \"Pneumonia\", \n",
    "\"Atelectasis\", \"Pneumothorax\", \"Pleural Effusion\", \"Pleural Other\", \"Fracture\", \"Support Devices\"] # without \"No Finding\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computes negative F1 and negative F1-5 for the labels:\n",
    "# Edema, Consolidation, Pneumonia, Pneumothorax, Pleural Effusion.\n",
    "# Also returns a list of Negative F1's for each label\n",
    "def negative_f1(gt, pred):\n",
    "    labels = range(13)\n",
    "    labels_five = list(map(lambda x: CXR_LABELS_1.index(x), \n",
    "                           [\"Edema\", \"Consolidation\", \"Pneumonia\", \n",
    "                            \"Pneumothorax\", \"Pleural Effusion\"]))\n",
    "    f1_scores = []\n",
    "\n",
    "    for i in labels:\n",
    "        score = f1_score(gt[:, i], pred[:, i], zero_division=0)\n",
    "        f1_scores.append(score)\n",
    "    f1_scores = np.array(f1_scores)\n",
    "\n",
    "    neg_f1 = f1_scores.mean()\n",
    "    neg_f1_five = f1_scores[labels_five].mean()\n",
    "    return neg_f1, neg_f1_five, f1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computes positive F1 and positive F1-5 for all labels except No Finding\n",
    "# When `use_five` is True, we only calculate F1 with the labels:\n",
    "# Atelectasis, Consolidation, Edema, Pleural Effusion, Cardiomegaly\n",
    "def positive_f1(gt, pred):\n",
    "    labels = range(13)\n",
    "    labels_five = list(map(lambda x: CXR_LABELS_1.index(x), \n",
    "                           [\"Cardiomegaly\", \"Edema\", \"Consolidation\", \n",
    "                            \"Atelectasis\", \"Pleural Effusion\"]))\n",
    "    f1_scores = []\n",
    "\n",
    "    for i in labels:\n",
    "        score = f1_score(gt[:, i], pred[:, i], zero_division=0)\n",
    "        f1_scores.append(score)\n",
    "    f1_scores = np.array(f1_scores)\n",
    "\n",
    "    pos_f1 = f1_scores.mean()\n",
    "    pos_f1_five = f1_scores[labels_five].mean()\n",
    "    return pos_f1, pos_f1_five, f1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computes the positive and negative F1 (excluding No Finding)\n",
    "def compute_f1(df_gt, df_pred):\n",
    "    y_gt = np.array(df_gt.drop(columns=[\"study_id\", \"No Finding\"]))\n",
    "\n",
    "    # Note on labels:\n",
    "    # 2: unmentioned ; 1: positive ; 0: negative ; -1: uncertain\n",
    "    y_gt_neg = y_gt.copy()\n",
    "    y_gt_neg[y_gt_neg == 0] = 3\n",
    "    y_gt_neg[(y_gt_neg == 1) | (y_gt_neg == 2) | (y_gt_neg == -1)] = 0\n",
    "    y_gt_neg[y_gt_neg == 3] = 1\n",
    "    \n",
    "    y_gt[(y_gt == -1) | (y_gt == 2)] = 0\n",
    "\n",
    "    y_pred = np.array(df_pred.drop(columns=[\"study_id\", \"No Finding\"]))\n",
    "\n",
    "    y_pred_neg = y_pred.copy()\n",
    "    y_pred_neg[y_pred_neg == 0] = 3\n",
    "    y_pred_neg[(y_pred_neg == 1) | (y_pred_neg == 2) | (y_pred_neg == -1)] = 0\n",
    "    y_pred_neg[y_pred_neg == 3] = 1\n",
    "    \n",
    "    y_pred[(y_pred == -1) | (y_pred == 2)] = 0\n",
    "\n",
    "    pos_f1, pos_f1_five, label_pos_f1 = positive_f1(y_gt, y_pred)\n",
    "    neg_f1, neg_f1_five, label_neg_f1 = negative_f1(y_gt_neg, y_pred_neg)\n",
    "    return pos_f1, pos_f1_five, neg_f1, neg_f1_five, label_pos_f1, label_neg_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 15)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gen = pd.read_csv(GEN_DIR+'gen_labels_sample_2_1.csv').sort_values(by='study_id').reset_index(drop=True)\n",
    "df_gen.dropna(inplace=True)\n",
    "df_gen.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 15)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gt = pd.read_csv(GT_DIR+\"gt_labels_sample.csv\").sort_values(by='study_id').reset_index(drop=True)\n",
    "df_gt.dropna(inplace=True)\n",
    "df_gt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 15)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gt = df_gt[df_gt['study_id'].isin(df_gen['study_id'])]\n",
    "df_gt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_temp = df_gt.drop(columns=[\"study_id\", \"No Finding\"])\n",
    "df_temp.columns == CXR_LABELS_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pos f1</th>\n",
       "      <td>0.145816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pos f1_5</th>\n",
       "      <td>0.208012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neg f1</th>\n",
       "      <td>0.091242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neg f1_5</th>\n",
       "      <td>0.209201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>0.055556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>0.260870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>0.172840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>0.206349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>0.466667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 2_1\n",
       "pos f1                      0.145816\n",
       "pos f1_5                    0.208012\n",
       "neg f1                      0.091242\n",
       "neg f1_5                    0.209201\n",
       "Enlarged Cardiomediastinum  0.055556\n",
       "Cardiomegaly                0.260870\n",
       "Lung Lesion                 0.000000\n",
       "Lung Opacity                0.333333\n",
       "Edema                       0.172840\n",
       "Consolidation               0.000000\n",
       "Pneumonia                   0.000000\n",
       "Atelectasis                 0.206349\n",
       "Pneumothorax                0.000000\n",
       "Pleural Effusion            0.400000\n",
       "Pleural Other               0.000000\n",
       "Fracture                    0.000000\n",
       "Support Devices             0.466667"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# F1 calculation\n",
    "pos_f1, pos_f1_five, neg_f1, neg_f1_five, label_pos_f1, label_neg_f1 = compute_f1(df_gt, df_gen)\n",
    "\n",
    "dict = {}\n",
    "dict['pos f1'] = pos_f1\n",
    "dict['pos f1_5'] = pos_f1_five\n",
    "dict['neg f1'] = neg_f1\n",
    "dict['neg f1_5'] = neg_f1_five\n",
    "for col in CXR_LABELS_1:\n",
    "    dict[col] = label_pos_f1[CXR_LABELS_1.index(col)]\n",
    "\n",
    "df_metrics1 = pd.DataFrame(dict, index=['2_1']).T\n",
    "df_metrics1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}