creation of labs input.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "92e86bae-7a34-44f7-aa22-b76d3020999c",
   "metadata": {},
   "source": [
    "#### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2640cb3b-8d28-487e-8d4b-15e931cb7c7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import datetime as dt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from matplotlib import pyplot\n",
    "from matplotlib.pyplot import figure"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bf24278-94f1-42f9-b9b8-13f523328858",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Read in files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66c0eefb-c4e9-4886-94e0-29863e071ca8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "labFilePath = r'C:\\\\Users\\\\momenzadeha\\\\Documents\\hypogly_pred\\EHR_extracts\\meyer_2306_labs_with_order_time_2023_03_06.csv'\n",
    "originalLabDf = pd.read_csv(labFilePath)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ce49e7-b3f4-4ef6-ba65-e4507904b380",
   "metadata": {},
   "source": [
    "#### Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0af17a9d-2653-41ca-be4c-d241f5b0d95b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#clean duplicate lab names\n",
    "replacement_dict = {\n",
    "    'GLUCOSE (POC)': 'GLUCOSE-POC',\n",
    "    'SERUM CREATININE': 'CREATININE',\n",
    "}\n",
    "originalLabDf = originalLabDf.replace(replacement_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd2f063d-5160-427b-a181-89d7401871ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract these labs only\n",
    "lab_list=['GLUCOSE-POC', 'CREATININE']\n",
    "targLabDf=originalLabDf[originalLabDf['COMPONENT_NAME'].isin(lab_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca8a8970-afe2-4e5b-95fd-321c01fa6c99",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to format the lab DataFrame and remove missing lab entries\n",
    "def format_lab_df(originalLabDf):\n",
    "    \"\"\"\n",
    "    This function formats the input lab DataFrame by:\n",
    "    - Renaming columns for consistency.\n",
    "    - Selecting relevant columns.\n",
    "    - Converting date columns to datetime format.\n",
    "    - Cleaning up erroneous values in the 'ORD_VALUE' column.\n",
    "    - Converting 'ORD_VALUE' to numeric and removing rows with missing 'ORD_VALUE' values.\n",
    "\n",
    "    Parameters:\n",
    "    originalLabDf (pd.DataFrame): The original lab data DataFrame.\n",
    "\n",
    "    Returns:\n",
    "    pd.DataFrame: A cleaned and formatted lab DataFrame.\n",
    "    \"\"\"\n",
    "    labDf = originalLabDf.copy()\n",
    "    labDf.rename(columns={'PAT_ENC_CSN_ID': 'CSN'}, inplace=True)\n",
    "    labDf = labDf[['MRN', 'CSN', 'CONTACT_DATE', 'RESULT_TIME', 'COMPONENT_NAME', 'ORD_VALUE']]\n",
    "    labDf[['CONTACT_DATE', 'RESULT_TIME']] = labDf[['CONTACT_DATE', 'RESULT_TIME']].apply(pd.to_datetime)\n",
    "    labDf['ORD_VALUE'].replace(\n",
    "        ['154bs', 'BS 175', 'bs120', '168 mg/dL', '159=bs', 'bs 120', '`185', '250 mg/dL', 'BG 171 mg/dL', \n",
    "         '250 mg/dL', '240 mg/dL', '130  mg/dL', 'BS 145', 'bs-134'], \n",
    "        ['154', '175', '120', '168', '159', '120', '185', '250', '171', '250', '240', '130', '145', '134'], \n",
    "        inplace=True)\n",
    "    labDf['ORD_VALUE'] = pd.to_numeric(labDf['ORD_VALUE'], errors='coerce')\n",
    "    # Drop rows where 'ORD_VALUE' is missing (NaN), as these are invalid or incomplete entries\n",
    "    labDf = labDf.dropna(subset=['ORD_VALUE'])\n",
    "    # Return the cleaned and formatted lab DataFrame\n",
    "    return labDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3deabe0e-2a9e-4ccc-bc16-2f577dee58b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "labDf = format_lab_df(targLabDf)\n",
    "filteredLabDf = labDf.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "791612be-b298-43fe-b38f-55f9d05168c5",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Calculate differences between lab times and random BG time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c722ce98-b166-4a43-b371-d5bd241499a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#CSNs to merge on from meds dataset\n",
    "with open(r'/Users/momenzadeha/Documents/mixed effects LR/LMM/MELRoutputCSNs.pkl', 'rb') as handle:\n",
    "    list_for_labs = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a4d5dcf6-2daf-4ba3-8b7d-3fb8423c7817",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_for_labs.rename(columns={'RESULT_TIME': 'rand_BG_time'}, inplace=True)\n",
    "list_for_labs = list_for_labs[['CSN', 'rand_BG_time']].drop_duplicates()\n",
    "# Perform an inner join between 'list_for_labs' and 'filteredLabDf' DataFrames on the 'CSN' column\n",
    "merge = list_for_labs.merge(filteredLabDf, how='inner', on='CSN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "68a2b24f-da72-492f-ad74-4033fd2944c6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_time_difference_column_in_lab_before_event_df(merge):\n",
    "    \"\"\"\n",
    "    This function creates a 'TIME_DIFF' column in the input DataFrame 'merge' which calculates the difference between \n",
    "    'rand_BG_time' and 'RESULT_TIME'. The time difference is converted into minutes using NumPy's timedelta64 format.\n",
    "    \n",
    "    Parameters:\n",
    "    merge (pd.DataFrame): A DataFrame containing 'rand_BG_time' and 'RESULT_TIME' columns that represent \n",
    "                          time of random blood glucose measurements and lab result times, respectively.\n",
    "    \n",
    "    Returns:\n",
    "    pd.DataFrame: The updated DataFrame with a new 'TIME_DIFF' column and a subset of the original columns.\n",
    "    \"\"\"   \n",
    "    diffFromLabColumn = merge['rand_BG_time'] - merge['RESULT_TIME']\n",
    "    diffFromLabColumn = [np.timedelta64(x, 'm') for x in diffFromLabColumn]\n",
    "    merge['TIME_DIFF'] = diffFromLabColumn\n",
    "    merge = merge[['CSN', 'rand_BG_time', 'RESULT_TIME', 'TIME_DIFF', 'COMPONENT_NAME', 'ORD_VALUE']]\n",
    "    # Return the updated DataFrame with the 'TIME_DIFF' column\n",
    "    return merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "f268c74a-821b-4d79-93cd-c53499834236",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_time_difference_column_in_lab_before_event_df(merge):\n",
    "    \"\"\"\n",
    "    This function creates a new column, 'TIME_DIFF', in the input DataFrame 'merge', which represents the time difference \n",
    "    between two columns: 'rand_BG_time' and 'RESULT_TIME'. The time difference is calculated in minutes ('m').\n",
    "\n",
    "    Parameters:\n",
    "    merge (pd.DataFrame): A pandas DataFrame containing the columns 'rand_BG_time' and 'RESULT_TIME', representing \n",
    "                          the time of a random blood glucose measurement and the time of a lab result, respectively.\n",
    "\n",
    "    Returns:\n",
    "    pd.DataFrame: The updated DataFrame with a new 'TIME_DIFF' column and a subset of the original columns.\n",
    "    \"\"\"\n",
    "    diffFromLabColumn = merge['rand_BG_time'] - merge['RESULT_TIME']\n",
    "    diffFromLabColumn = [np.timedelta64(x, 'm') for x in diffFromLabColumn]\n",
    "    merge['TIME_DIFF'] = diffFromLabColumn\n",
    "    merge = merge[['CSN', 'rand_BG_time', 'RESULT_TIME', 'TIME_DIFF', 'COMPONENT_NAME', 'ORD_VALUE']]\n",
    "    return merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9eaa3b35-c74e-4f9a-b35d-7a551948dd38",
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_diff = create_time_difference_column_in_lab_before_event_df(merge)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90ed066d-2b50-4821-bdc4-56df8cd60277",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Filter for labs only in lookback window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ce9454d9-7305-430e-ba76-efcba19da01f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def establish_random_delay_before_lab_test_for_each_csn(csnRts, delayTimeLength):\n",
    "    \"\"\"\n",
    "    This function creates a dictionary of random delays for each unique CSN and RESULT_TIME pair.\n",
    "    The delay is constant for all pairs in this implementation.\n",
    "\n",
    "    Parameters:\n",
    "    csnRts (set): A set of tuples where each tuple contains a unique combination of CSN and RESULT_TIME.\n",
    "    delayTimeLength (int): The constant random delay (in hours) to be assigned to each CSN and RESULT_TIME pair.\n",
    "\n",
    "    Returns:\n",
    "    dict: A dictionary where keys are tuples of (CSN, RESULT_TIME) and values are the specified random delay.\n",
    "    \"\"\"\n",
    "    # Create a dictionary where each key (CSN, RESULT_TIME pair) is assigned the same delay time length\n",
    "    return {key: delayTimeLength for key in csnRts}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "fe59536e-412a-4999-972d-92ba71832e04",
   "metadata": {},
   "outputs": [],
   "source": [
    "def divide_times_into_segments_for_each_csn_result_time(csnRt, timeDiffs, randomDelayDict, numSections, timeWindowHours):\n",
    "    \"\"\"\n",
    "    This function divides the time differences ('timeDiffs') for a specific CSN (patient encounter) into segments.\n",
    "    The time window is adjusted based on the random delay before the lab test and a specified number of sections\n",
    "    over a customizable time window (in hours).\n",
    "\n",
    "    Parameters:\n",
    "    csnRt (tuple): A tuple containing a unique identifier for the CSN (Clinical Serial Number) and RESULT_TIME.\n",
    "    timeDiffs (list): A list of time differences between random BG measurements and lab results (in minutes).\n",
    "    randomDelayDict (dict): A dictionary mapping each CSN to a random delay value before the lab test.\n",
    "    numSections (int): The number of segments to divide the time window into.\n",
    "    timeWindowHours (int): The total time window in hours to create segments.\n",
    "\n",
    "    Returns:\n",
    "    list: A list of segment categories indicating in which segment each time difference falls.\n",
    "    \"\"\"\n",
    "    endTime = np.timedelta64(int(np.round(randomDelayDict[csnRt] * 60)), 'm')\n",
    "    startTime = endTime + np.timedelta64(timeWindowHours * 60, 'm')\n",
    "    # Generate 'numSections + 1' equally spaced cutoffs between endTime and startTime\n",
    "    # These cutoffs define the boundaries of each time segment\n",
    "    segmentCutoffs = np.linspace(endTime.astype(int), startTime.astype(int), numSections + 1)\n",
    "    segmentCutoffs = [np.timedelta64(int(x), 'm') for x in segmentCutoffs]\n",
    "    timeDiffs = [np.timedelta64(x, 'm') for x in timeDiffs]\n",
    "    # Categorize each time difference into one of the time segments based on the segment cutoffs\n",
    "    # np.searchsorted() returns the indices of the segment that each time difference falls into\n",
    "    segmentCategories = np.searchsorted(segmentCutoffs, timeDiffs, side='right')\n",
    "    # Return the list of segment categories for each time difference\n",
    "    return segmentCategories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "527f196e-6b15-454a-8bd5-ed3d12f43083",
   "metadata": {},
   "outputs": [],
   "source": [
    "def identify_time_section_window_from_lab_result(merge, numSections, randomDelayMaxLength, timeWindowHours):\n",
    "    \"\"\"\n",
    "    This function identifies and assigns time segments for each row in the dataframe based on time differences between \n",
    "    random BG measurements and lab results, considering a random delay for each CSN (patient encounter).\n",
    "\n",
    "    Parameters:\n",
    "    merge (DataFrame): The input dataframe containing CSN, RESULT_TIME, TIME_DIFF, and other relevant columns.\n",
    "    numSections (int): The number of segments to divide the time window into.\n",
    "    randomDelayMaxLength (int): The maximum length (in hours) of the random delay before lab tests for each CSN.\n",
    "    timeWindowHours (int): The total time window (in hours) to be segmented into sections.\n",
    "\n",
    "    Returns:\n",
    "    DataFrame: A copy of the original dataframe with an added 'SEGMENT' column that indicates the time segment for each row.\n",
    "    dict: A dictionary containing the random delay values for each CSN and RESULT_TIME pair.\n",
    "    \"\"\"\n",
    "    merge = merge.copy()\n",
    "    merge = merge.sort_values(['CSN', 'RESULT_TIME', 'TIME_DIFF']).reset_index(drop=True)\n",
    "    # Establish a dictionary of random delays for each unique CSN and RESULT_TIME pair.\n",
    "    # The random delay is generated with a maximum length defined by 'randomDelayMaxLength'.\n",
    "    randomDelayDict = establish_random_delay_before_lab_test_for_each_csn(\n",
    "        set(zip(merge['CSN'], merge['RESULT_TIME'])), randomDelayMaxLength\n",
    "    )\n",
    "    # Apply the time segmentation to each group of rows with the same CSN and RESULT_TIME.\n",
    "    # The 'transform' function applies the segmentation for each group.\n",
    "    merge['SEGMENT'] = merge.groupby(['CSN', 'RESULT_TIME'])['TIME_DIFF'].transform(\n",
    "        lambda timeDiffs: divide_times_into_segments_for_each_csn_result_time(\n",
    "            timeDiffs.name, timeDiffs, randomDelayDict, numSections, timeWindowHours))\n",
    "    # Return the modified dataframe with the 'SEGMENT' column and the random delay dictionary\n",
    "    return merge, randomDelayDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f8d1607f-b43e-430c-8de8-72e63d1555ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "numSegments = 1  # Number of segments to divide the time window into\n",
    "randomDelayMaxLength = 4  # Maximum length (in hours) for the random delay before the lab test\n",
    "timeWindowHours = 24  # Total time window (in hours) over which to segment the time differences\n",
    "random.seed(0)\n",
    "merge_seg, randomDelayDict = identify_time_section_window_from_lab_result(\n",
    "    merge_diff,\n",
    "    numSegments, \n",
    "    randomDelayMaxLength, \n",
    "    timeWindowHours  \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "995c6baa-c539-420e-bc0b-36823dd82431",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_labs_data(merge_seg):\n",
    "    # Filter out rows where SEGMENT is 0 or 2\n",
    "    labs_red = merge_seg[(merge_seg.SEGMENT != 0) & (merge_seg.SEGMENT != 2)]\n",
    "    labs_red_sorted = labs_red.sort_values(by=['CSN', 'COMPONENT_NAME', 'RESULT_TIME'], ascending=[True, True, False])\n",
    "    # Group by CSN and COMPONENT_NAME, select the first entry\n",
    "    labs_red_grouped = labs_red_sorted.groupby(['CSN', 'COMPONENT_NAME']).first().reset_index()\n",
    "    def create_pivot(df, agg_func, suffix):\n",
    "        return (df.groupby(['CSN', 'COMPONENT_NAME'])\n",
    "                .agg({'ORD_VALUE': agg_func})\n",
    "                .pipe(lambda df: pd.pivot_table(df, values='ORD_VALUE', index='CSN', columns='COMPONENT_NAME'))\n",
    "                .rename(columns=lambda col: f'{col}_{suffix}'))\n",
    "    labs_red_last_piv = pd.pivot_table(labs_red_grouped, values='ORD_VALUE', index='CSN', columns='COMPONENT_NAME')\n",
    "    labs_red_last_piv.columns = [f'{col}_last' for col in labs_red_last_piv.columns]\n",
    "    labs_red_agg_mean_piv = create_pivot(labs_red, 'mean', 'mean')\n",
    "    labs_red_agg_min_piv = create_pivot(labs_red, 'min', 'min')\n",
    "    labs_red_agg_max_piv = create_pivot(labs_red, 'max', 'max')\n",
    "    # Merge all pivot tables\n",
    "    labs_merged = (labs_red_agg_mean_piv\n",
    "                    .join(labs_red_agg_min_piv, how='inner')\n",
    "                    .join(labs_red_agg_max_piv, how='inner')\n",
    "                    .join(labs_red_last_piv, how='inner'))\n",
    "    return labs_merged.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d473a787-1ee2-45a6-b7f3-c40b30c27137",
   "metadata": {},
   "outputs": [],
   "source": [
    "labs_merged = aggregate_labs_data(merge_seg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5ba3e1be-5d36-43ce-98f5-6644d8cc833b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'/Users/momenzadeha/Documents/mixed effects LR/LMM/sel_labs_24h.pkl', 'wb') as handle:\n",
    "    pickle.dump(labs_merged, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "base"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}