2_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Dependencies__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy  # copy big/deep objects by value\n",
    "import datetime  # datetime operations\n",
    "import itertools  # operate with iterators\n",
    "import json  # read/write from/into json format\n",
    "import os  # OS operations (read/write files/folders)\n",
    "import warnings  # hide warnings\n",
    "# process parallelization\n",
    "from multiprocessing import Lock, Manager, Pool, RawArray, cpu_count, managers\n",
    "\n",
    "import matplotlib.patches as mpatches  # create custom plot legends\n",
    "import matplotlib.pyplot as plt  # mother of plots for Python\n",
    "import matplotlib.ticker as ticker  # matplotlib ticker utils\n",
    "import numpy as np  # array/matrix operations (e.g. linear algebra)\n",
    "import pandas as pd  # operate with dataframes\n",
    "import seaborn as sns  # matplotlib plotting nice with shortcuts\n",
    "from IPython.display import display  # print nicely\n",
    "from matplotlib.lines import Line2D  # matplotlib custom lines\n",
    "from scipy import stats  # statistical tests\n",
    "from tqdm.notebook import tqdm  # mother of progressbars for Python\n",
    "\n",
    "# from matplotlib.ticker import FormatStrFormatter  # tick formatter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Options and constants__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total CPU threads: 16\n"
     ]
    }
   ],
   "source": [
    "warnings.filterwarnings(\"ignore\")\n",
    "# set default float display format to 2 decimals\n",
    "pd.options.display.float_format = \"{:.3f}\".format\n",
    "\n",
    "PATH_PROC = \"./data/processed\"\n",
    "\n",
    "# progress bar customized format\n",
    "B_FORMAT = \"\"\"📄 {n_fmt} of {total_fmt} {desc} processed: {bar} \n",
    "            {percentage:3.0f}% ⏱️{elapsed} ⏳{remaining} ⚙️{rate_fmt}{postfix}\"\"\"\n",
    "\n",
    "\"\"\"\n",
    "Event1: 1162-1400\n",
    "Event2: 1958-2080\n",
    "Event3: 2596-2890\n",
    "\"\"\"\n",
    "event_ranges = [(1162, 1400), (1958, 2080), (2596, 2890)]\n",
    "\n",
    "CORES = cpu_count()  # number of cpu threads for multiprocessing\n",
    "print(f\"Total CPU threads: {CORES}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Helper functions__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pbar_fork_hack(l=None):\n",
    "    \"\"\"\n",
    "    Hack to enforce progress bars to be displayed by fork processes on\n",
    "    IPython Apps like Jupyter Notebooks.\n",
    "\n",
    "    Avoids [IPKernelApp] WARNING | WARNING: attempted to send message from fork\n",
    "\n",
    "    Important: pass this function as argument for the initializer parameter\n",
    "    while initializing a multiprocessing pool to make it work. E.g.:\n",
    "\n",
    "    pool = Pool(processes=N_CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "    Source:\n",
    "     - https://github.com/ipython/ipython/issues/11049#issue-306086846\n",
    "     - https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308\n",
    "    \"\"\"\n",
    "    if l:\n",
    "        global lock\n",
    "        lock = l\n",
    "    print(\" \", end=\"\", flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Read participants cleaned data__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>expo</th>\n",
       "      <th>side</th>\n",
       "      <th>condition</th>\n",
       "      <th>nulls_%</th>\n",
       "      <th>sky_%</th>\n",
       "      <th>insidecar_%</th>\n",
       "      <th>questionnaire</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>2019-05-14 13:21:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>1.610</td>\n",
       "      <td>13.740</td>\n",
       "      <td>0.487</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>2019-05-14 16:44:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.674</td>\n",
       "      <td>0.000</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>2019-05-14 16:51:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>RadioTalk</td>\n",
       "      <td>0.037</td>\n",
       "      <td>33.134</td>\n",
       "      <td>0.000</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>2019-05-14 16:54:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>left</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>0.824</td>\n",
       "      <td>18.607</td>\n",
       "      <td>0.374</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>2019-05-14 16:56:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>0.225</td>\n",
       "      <td>26.994</td>\n",
       "      <td>0.000</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>2019-12-30 15:29:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>0.300</td>\n",
       "      <td>32.160</td>\n",
       "      <td>0.000</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>2020-01-03 16:12:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>0.674</td>\n",
       "      <td>15.462</td>\n",
       "      <td>0.037</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>2020-01-03 16:13:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>left</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>1.797</td>\n",
       "      <td>3.370</td>\n",
       "      <td>0.150</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>2020-01-03 16:15:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>left</td>\n",
       "      <td>RadioTalk</td>\n",
       "      <td>0.150</td>\n",
       "      <td>17.484</td>\n",
       "      <td>0.187</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>2020-01-03 16:16:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>0.187</td>\n",
       "      <td>2.583</td>\n",
       "      <td>1.011</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 date  expo   side  \\\n",
       "id                                                                   \n",
       "4f583872ffed48249874b00f3d389cfc  2019-05-14 13:21:00   MSW  right   \n",
       "33d68ad55ef544fab9fd08108e3913ca  2019-05-14 16:44:00   MSW  right   \n",
       "8598a83e2af441b8bcd0ae5d84beb875  2019-05-14 16:51:00   MSW  right   \n",
       "37d7e67934974217830bb429bba7fd76  2019-05-14 16:54:00   MSW   left   \n",
       "42bac596059749b5b8e8e83ae61de9b4  2019-05-14 16:56:00   MSW  right   \n",
       "...                                               ...   ...    ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94  2019-12-30 15:29:00  BMBF  right   \n",
       "611d35a7c3e940cc82495e53c2a8532d  2020-01-03 16:12:00  BMBF  right   \n",
       "3b6fda285d9e412eb081986b2f22a2e3  2020-01-03 16:13:00  BMBF   left   \n",
       "18ffb0abdc8642098c479380bfa533d1  2020-01-03 16:15:00  BMBF   left   \n",
       "c06f123b35b74bb489ec239b1cac9eb4  2020-01-03 16:16:00  BMBF  right   \n",
       "\n",
       "                                   condition  nulls_%  sky_%  insidecar_%  \\\n",
       "id                                                                          \n",
       "4f583872ffed48249874b00f3d389cfc  TaxiDriver    1.610 13.740        0.487   \n",
       "33d68ad55ef544fab9fd08108e3913ca        AVAS    0.037  0.674        0.000   \n",
       "8598a83e2af441b8bcd0ae5d84beb875   RadioTalk    0.037 33.134        0.000   \n",
       "37d7e67934974217830bb429bba7fd76        AVAS    0.824 18.607        0.374   \n",
       "42bac596059749b5b8e8e83ae61de9b4  TaxiDriver    0.225 26.994        0.000   \n",
       "...                                      ...      ...    ...          ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94        AVAS    0.300 32.160        0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d  TaxiDriver    0.674 15.462        0.037   \n",
       "3b6fda285d9e412eb081986b2f22a2e3        AVAS    1.797  3.370        0.150   \n",
       "18ffb0abdc8642098c479380bfa533d1   RadioTalk    0.150 17.484        0.187   \n",
       "c06f123b35b74bb489ec239b1cac9eb4  TaxiDriver    0.187  2.583        1.011   \n",
       "\n",
       "                                 questionnaire  \n",
       "id                                              \n",
       "4f583872ffed48249874b00f3d389cfc            No  \n",
       "33d68ad55ef544fab9fd08108e3913ca            No  \n",
       "8598a83e2af441b8bcd0ae5d84beb875            No  \n",
       "37d7e67934974217830bb429bba7fd76            No  \n",
       "42bac596059749b5b8e8e83ae61de9b4            No  \n",
       "...                                        ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94            No  \n",
       "611d35a7c3e940cc82495e53c2a8532d            No  \n",
       "3b6fda285d9e412eb081986b2f22a2e3            No  \n",
       "18ffb0abdc8642098c479380bfa533d1           Yes  \n",
       "c06f123b35b74bb489ec239b1cac9eb4            No  \n",
       "\n",
       "[24356 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parts = pd.read_csv(\"./participants_clean.csv\")\n",
    "parts = parts.set_index(\"id\")  # set uid column as index (remove default)\n",
    "parts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect all gazes and hit frames on critical events "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de6d1e05118945ffbd94e401c2b34a82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 4 📂 proxy objects processed:                                                                           …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95d952b8456f4ff2aff7492e950d1582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def events_frames_all(uid):\n",
    "    \"\"\"\n",
    "    Collect critical event frames of all participants.\n",
    "    'h_start' -> 1st frame number, all hits\n",
    "    'g_start' -> 1st frame number, all gazes\n",
    "    'h_all'   -> All frames, all hits\n",
    "    'g_all'   -> All frames, all gazes\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant unique identifier.\n",
    "    \"\"\"\n",
    "    # load event data\n",
    "    events = pd.read_csv(f\"{PATH_PROC}/hit_events/{uid}.csv\")\n",
    "    for objct in objcts:\n",
    "        # select all hits of the current event\n",
    "        query = events[\"name\"].str.contains(objct, regex=False, na=False)\n",
    "        h_events = events[query]\n",
    "        starts = h_events.start.tolist()\n",
    "        ends = h_events.end.tolist()\n",
    "        # store list of start hits (any)\n",
    "        if len(starts) > 0:\n",
    "            # store hit starts\n",
    "            for f in starts:\n",
    "                lock.acquire()\n",
    "                res[\"h_start\"][objct].append(f)\n",
    "                lock.release()\n",
    "            # collect all hits\n",
    "            frames = [\n",
    "                [i for i in range(s, e + 1)] for s, e in zip(starts, ends)\n",
    "            ]\n",
    "            # flatten list\n",
    "            frames = [h for hlist in frames for h in hlist]\n",
    "            # store all hits\n",
    "            for f in frames:\n",
    "                lock.acquire()\n",
    "                res[\"h_all\"][objct].append(f)\n",
    "                lock.release()\n",
    "            # select only gaze events\n",
    "            h_events = h_events[h_events[\"type\"] == \"gaze\"]\n",
    "            starts = h_events.start.tolist()\n",
    "            ends = h_events.end.tolist()\n",
    "            # store list of start hits (gazes)\n",
    "            for f in starts:\n",
    "                lock.acquire()\n",
    "                res[\"g_start\"][objct].append(f)\n",
    "                lock.release()\n",
    "            # collect all hits\n",
    "            frames = [\n",
    "                [i for i in range(s, e + 1)] for s, e in zip(starts, ends)\n",
    "            ]\n",
    "            # flatten list\n",
    "            frames = [h for hlist in frames for h in hlist]\n",
    "            # store all gaze hits\n",
    "            for f in frames:\n",
    "                lock.acquire()\n",
    "                res[\"g_all\"][objct].append(f)\n",
    "                lock.release()\n",
    "\n",
    "\n",
    "manager = Manager()  # manage shared memory types\n",
    "\n",
    "# event names and hit/gaze groups to build dict\n",
    "objcts = [\"jogging_man\", \"fast_car\", \"walking_man\"]\n",
    "h_groups = [\"h_start\", \"h_all\", \"g_start\", \"g_all\"]\n",
    "\n",
    "# proxy object progress bar\n",
    "proxy_pb = tqdm(\n",
    "    iterable=h_groups,\n",
    "    total=len(h_groups),\n",
    "    desc=\"📂 proxy objects\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# convert into memory-shared dict of dicts for multiprocessing\n",
    "res = manager.dict(\n",
    "    {g: manager.dict({o: manager.list() for o in objcts}) for g in proxy_pb}\n",
    ")\n",
    "\n",
    "l = Lock()\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack, initargs=(l,))\n",
    "\n",
    "# participants ids\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap_unordered(func=events_frames_all, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make proxy object readable and store it as a JSON dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decode proxy object\n",
    "# we include the filter (e < 2000 and c == \"walking_man\") since due to\n",
    "# Farbod's NPC's random respawn, the walking man appeared during the\n",
    "# instructions of the experiment and was hit\n",
    "temp = dict(res)\n",
    "temp = {k: dict(v) for k, v in temp.items()}\n",
    "temp = {\n",
    "    k: {\n",
    "        c: [e for e in list(b) if not (e < 2000 and c == \"walking_man\")]\n",
    "        for c, b in v.items()\n",
    "    }\n",
    "    for k, v in temp.items()\n",
    "}\n",
    "\n",
    "# store it\n",
    "with open(\"gazes_and_hits-event_frames.json\", \"w\") as file:\n",
    "    json.dump(temp, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect gazes and hit frames on critical events per participant (individual files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8822338058ba44be8d09990f6ca28f2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def events_frames_part(uid):\n",
    "    \"\"\"\n",
    "    Collect and store critical event frames of each participant.\n",
    "    'h_start' -> 1st frame number, all hits\n",
    "    'g_start' -> 1st frame number, all gazes\n",
    "    'h_all'   -> All frames, all hits\n",
    "    'g_all'   -> All frames, all gazes\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant unique identifier.\n",
    "    \"\"\"\n",
    "    # dict to collect hit and gaze frames\n",
    "    res = {o: {g: [] for g in h_groups} for o in objcts}\n",
    "    # load event data\n",
    "    events = pd.read_csv(f\"{PATH_PROC}/hit_events/{uid}.csv\")\n",
    "    for objct in objcts:\n",
    "        # select all hits of the current event\n",
    "        query = events[\"name\"].str.contains(objct, regex=False, na=False)\n",
    "        h_events = events[query]\n",
    "        starts = h_events.start.tolist()\n",
    "        ends = h_events.end.tolist()\n",
    "        # store list of start hits (any)\n",
    "        if len(starts) > 0:\n",
    "            # store hit starts\n",
    "            for f in starts:\n",
    "                res[objct][\"h_start\"].append(f)\n",
    "            # collect all hits\n",
    "            frames = [\n",
    "                [i for i in range(s, e + 1)] for s, e in zip(starts, ends)\n",
    "            ]\n",
    "            # flatten list\n",
    "            frames = [h for hlist in frames for h in hlist]\n",
    "            # store all hits\n",
    "            for f in frames:\n",
    "                res[objct][\"h_all\"].append(f)\n",
    "            # select only gaze events\n",
    "            h_events = h_events[h_events[\"type\"] == \"gaze\"]\n",
    "            starts = h_events.start.tolist()\n",
    "            ends = h_events.end.tolist()\n",
    "            # store list of start hits (gazes)\n",
    "            for f in starts:\n",
    "                res[objct][\"g_start\"].append(f)\n",
    "            # collect all hits\n",
    "            frames = [\n",
    "                [i for i in range(s, e + 1)] for s, e in zip(starts, ends)\n",
    "            ]\n",
    "            # flatten list\n",
    "            frames = [h for hlist in frames for h in hlist]\n",
    "            # store all gaze hits\n",
    "            for f in frames:\n",
    "                res[objct][\"g_all\"].append(f)\n",
    "\n",
    "            # store results\n",
    "            fname = f\"{PATH_PROC}/events/frames/{uid}.json\"\n",
    "            with open(fname, \"w\") as file:\n",
    "                json.dump(res, file)\n",
    "\n",
    "\n",
    "# event names and hit/gaze groups to build dict\n",
    "objcts = [\"jogging_man\", \"fast_car\", \"walking_man\"]\n",
    "h_groups = [\"h_start\", \"h_all\", \"g_start\", \"g_all\"]\n",
    "\n",
    "# participants ids\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap_unordered(func=events_frames_part, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buildings</th>\n",
       "      <th>Dynamic Cars</th>\n",
       "      <th>Dynamic Pedestrians</th>\n",
       "      <th>Inside Car</th>\n",
       "      <th>Nature</th>\n",
       "      <th>Roads</th>\n",
       "      <th>Sky</th>\n",
       "      <th>Static Cars</th>\n",
       "      <th>Static Pedestrians</th>\n",
       "      <th>Street Objects</th>\n",
       "      <th>Traffic Lights</th>\n",
       "      <th>Traffic Signs</th>\n",
       "      <th>nulls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>730</td>\n",
       "      <td>42</td>\n",
       "      <td>71</td>\n",
       "      <td>120</td>\n",
       "      <td>303</td>\n",
       "      <td>625</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "      <td>225</td>\n",
       "      <td>414</td>\n",
       "      <td>41</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>29</td>\n",
       "      <td>22</td>\n",
       "      <td>11</td>\n",
       "      <td>1254</td>\n",
       "      <td>26</td>\n",
       "      <td>1151</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>152</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>1245</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>77</td>\n",
       "      <td>13</td>\n",
       "      <td>875</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>297</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>799</td>\n",
       "      <td>84</td>\n",
       "      <td>89</td>\n",
       "      <td>224</td>\n",
       "      <td>383</td>\n",
       "      <td>55</td>\n",
       "      <td>9</td>\n",
       "      <td>45</td>\n",
       "      <td>27</td>\n",
       "      <td>295</td>\n",
       "      <td>587</td>\n",
       "      <td>48</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>1388</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>73</td>\n",
       "      <td>313</td>\n",
       "      <td>16</td>\n",
       "      <td>289</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>216</td>\n",
       "      <td>347</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>1270</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>241</td>\n",
       "      <td>144</td>\n",
       "      <td>0</td>\n",
       "      <td>601</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>263</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>841</td>\n",
       "      <td>13</td>\n",
       "      <td>35</td>\n",
       "      <td>856</td>\n",
       "      <td>289</td>\n",
       "      <td>112</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>294</td>\n",
       "      <td>26</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>367</td>\n",
       "      <td>11</td>\n",
       "      <td>80</td>\n",
       "      <td>788</td>\n",
       "      <td>93</td>\n",
       "      <td>667</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>336</td>\n",
       "      <td>206</td>\n",
       "      <td>14</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>740</td>\n",
       "      <td>21</td>\n",
       "      <td>45</td>\n",
       "      <td>304</td>\n",
       "      <td>391</td>\n",
       "      <td>259</td>\n",
       "      <td>170</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>246</td>\n",
       "      <td>401</td>\n",
       "      <td>66</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>58</td>\n",
       "      <td>94</td>\n",
       "      <td>24</td>\n",
       "      <td>1490</td>\n",
       "      <td>99</td>\n",
       "      <td>582</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>108</td>\n",
       "      <td>127</td>\n",
       "      <td>78</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Buildings  Dynamic Cars  \\\n",
       "4f583872ffed48249874b00f3d389cfc        730            42   \n",
       "33d68ad55ef544fab9fd08108e3913ca         29            22   \n",
       "8598a83e2af441b8bcd0ae5d84beb875       1245             2   \n",
       "37d7e67934974217830bb429bba7fd76        799            84   \n",
       "42bac596059749b5b8e8e83ae61de9b4       1388             0   \n",
       "...                                     ...           ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94       1270             0   \n",
       "611d35a7c3e940cc82495e53c2a8532d        841            13   \n",
       "3b6fda285d9e412eb081986b2f22a2e3        367            11   \n",
       "18ffb0abdc8642098c479380bfa533d1        740            21   \n",
       "c06f123b35b74bb489ec239b1cac9eb4         58            94   \n",
       "\n",
       "                                  Dynamic Pedestrians  Inside Car  Nature  \\\n",
       "4f583872ffed48249874b00f3d389cfc                   71         120     303   \n",
       "33d68ad55ef544fab9fd08108e3913ca                   11        1254      26   \n",
       "8598a83e2af441b8bcd0ae5d84beb875                    0          61      77   \n",
       "37d7e67934974217830bb429bba7fd76                   89         224     383   \n",
       "42bac596059749b5b8e8e83ae61de9b4                    6          73     313   \n",
       "...                                               ...         ...     ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94                    4         241     144   \n",
       "611d35a7c3e940cc82495e53c2a8532d                   35         856     289   \n",
       "3b6fda285d9e412eb081986b2f22a2e3                   80         788      93   \n",
       "18ffb0abdc8642098c479380bfa533d1                   45         304     391   \n",
       "c06f123b35b74bb489ec239b1cac9eb4                   24        1490      99   \n",
       "\n",
       "                                  Roads  Sky  Static Cars  Static Pedestrians  \\\n",
       "4f583872ffed48249874b00f3d389cfc    625    3           20                  13   \n",
       "33d68ad55ef544fab9fd08108e3913ca   1151    0            0                   0   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     13  875            0                   0   \n",
       "37d7e67934974217830bb429bba7fd76     55    9           45                  27   \n",
       "42bac596059749b5b8e8e83ae61de9b4     16  289            5                   0   \n",
       "...                                 ...  ...          ...                 ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94      0  601            0                   0   \n",
       "611d35a7c3e940cc82495e53c2a8532d    112    3            0                   0   \n",
       "3b6fda285d9e412eb081986b2f22a2e3    667   18            2                  35   \n",
       "18ffb0abdc8642098c479380bfa533d1    259  170            3                  18   \n",
       "c06f123b35b74bb489ec239b1cac9eb4    582    2            1                   0   \n",
       "\n",
       "                                  Street Objects  Traffic Lights  \\\n",
       "4f583872ffed48249874b00f3d389cfc             225             414   \n",
       "33d68ad55ef544fab9fd08108e3913ca              25             152   \n",
       "8598a83e2af441b8bcd0ae5d84beb875             100             297   \n",
       "37d7e67934974217830bb429bba7fd76             295             587   \n",
       "42bac596059749b5b8e8e83ae61de9b4             216             347   \n",
       "...                                          ...             ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94             140             263   \n",
       "611d35a7c3e940cc82495e53c2a8532d             182             294   \n",
       "3b6fda285d9e412eb081986b2f22a2e3             336             206   \n",
       "18ffb0abdc8642098c479380bfa533d1             246             401   \n",
       "c06f123b35b74bb489ec239b1cac9eb4             108             127   \n",
       "\n",
       "                                  Traffic Signs  nulls  \n",
       "4f583872ffed48249874b00f3d389cfc             41     64  \n",
       "33d68ad55ef544fab9fd08108e3913ca              0      1  \n",
       "8598a83e2af441b8bcd0ae5d84beb875              0      1  \n",
       "37d7e67934974217830bb429bba7fd76             48     26  \n",
       "42bac596059749b5b8e8e83ae61de9b4             12      6  \n",
       "...                                         ...    ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94              0      8  \n",
       "611d35a7c3e940cc82495e53c2a8532d             26     20  \n",
       "3b6fda285d9e412eb081986b2f22a2e3             14     54  \n",
       "18ffb0abdc8642098c479380bfa533d1             66      7  \n",
       "c06f123b35b74bb489ec239b1cac9eb4             78      8  \n",
       "\n",
       "[24356 rows x 13 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def hits_per_group(uid):\n",
    "    \"\"\"\n",
    "    Compute hits counts per object group of a participant given its UID.\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant unique identifier.\n",
    "    \"\"\"\n",
    "    # load participant data\n",
    "    h_events = pd.read_csv(f\"{PATH_PROC}/hit_events/{uid}.csv\")\n",
    "\n",
    "    # all groups hit (get rid of NaNs)\n",
    "    groups = h_events.group.unique().tolist()\n",
    "    groups = [g for g in groups if not pd.isnull(g)]\n",
    "\n",
    "    total = 0\n",
    "    # compute hit counts\n",
    "    for g in groups:\n",
    "        sel = h_events[h_events.group == g]\n",
    "        cnt = sel[\"nHits\"].sum()\n",
    "        grp_cnt[uid][g] = cnt\n",
    "        total += cnt\n",
    "\n",
    "    # include hit null counts\n",
    "    nulls = 2671 - total\n",
    "    grp_cnt[uid][\"nulls\"] = nulls\n",
    "\n",
    "    # add 0 counters to non-hit groups\n",
    "    for g in grp_cats:\n",
    "        if g not in groups and g != \"nulls\":\n",
    "            grp_cnt[uid][g] = 0\n",
    "\n",
    "\n",
    "# participants ids\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "manager = Manager()  # manage shared memory types\n",
    "\n",
    "# all possible groups\n",
    "grp_cats = [\n",
    "    \"Buildings\",\n",
    "    \"Dynamic Cars\",\n",
    "    \"Dynamic Pedestrians\",\n",
    "    \"Inside Car\",\n",
    "    \"Nature\",\n",
    "    \"Roads\",\n",
    "    \"Sky\",\n",
    "    \"Static Cars\",\n",
    "    \"Static Pedestrians\",\n",
    "    \"Street Objects\",\n",
    "    \"Traffic Lights\",\n",
    "    \"Traffic Signs\",\n",
    "    \"nulls\",\n",
    "]\n",
    "# arrange into a dict\n",
    "grp_stats = {cat: None for cat in grp_cats}\n",
    "\n",
    "# convert into memory-shared dict of dicts for multiprocessing\n",
    "grp_cnt = manager.dict({uid: manager.dict(grp_stats) for uid in uids})\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap_unordered(func=hits_per_group, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()\n",
    "\n",
    "# parse, store and preview collected data\n",
    "grp_cnt = dict(grp_cnt)\n",
    "grp_cnt = {k: dict(v) for k, v in grp_cnt.items()}\n",
    "grp_cnt = pd.DataFrame(grp_cnt).transpose()\n",
    "grp_cnt.to_csv(\"./group_hit_counts.csv\")\n",
    "display(grp_cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute gaze vs rest stats and store gaze length and gaze counts (events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "496a4226fa49456085feb94a64e966e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rest_cnt</th>\n",
       "      <th>gaze_cnt</th>\n",
       "      <th>rest_avg</th>\n",
       "      <th>gaze_avg</th>\n",
       "      <th>rest_mdn</th>\n",
       "      <th>gaze_mdn</th>\n",
       "      <th>rest_dist_avg</th>\n",
       "      <th>gaze_dist_avg</th>\n",
       "      <th>rest_dist_mdn</th>\n",
       "      <th>gaze_dist_mdn</th>\n",
       "      <th>rest_len</th>\n",
       "      <th>gaze_len</th>\n",
       "      <th>rest_l_%</th>\n",
       "      <th>gaze_l_%</th>\n",
       "      <th>rest_c_%</th>\n",
       "      <th>gaze_c_%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>434.000</td>\n",
       "      <td>67.000</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.876</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.495</td>\n",
       "      <td>104.681</td>\n",
       "      <td>100.493</td>\n",
       "      <td>85.326</td>\n",
       "      <td>66.196</td>\n",
       "      <td>28.644</td>\n",
       "      <td>58.707</td>\n",
       "      <td>32.792</td>\n",
       "      <td>67.208</td>\n",
       "      <td>86.627</td>\n",
       "      <td>13.373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>280.000</td>\n",
       "      <td>80.000</td>\n",
       "      <td>0.083</td>\n",
       "      <td>0.811</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.611</td>\n",
       "      <td>20.270</td>\n",
       "      <td>10.064</td>\n",
       "      <td>2.389</td>\n",
       "      <td>2.369</td>\n",
       "      <td>23.232</td>\n",
       "      <td>64.911</td>\n",
       "      <td>26.357</td>\n",
       "      <td>73.643</td>\n",
       "      <td>77.778</td>\n",
       "      <td>22.222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>110.000</td>\n",
       "      <td>37.000</td>\n",
       "      <td>0.089</td>\n",
       "      <td>1.336</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.660</td>\n",
       "      <td>95.261</td>\n",
       "      <td>148.664</td>\n",
       "      <td>47.726</td>\n",
       "      <td>92.824</td>\n",
       "      <td>9.834</td>\n",
       "      <td>49.434</td>\n",
       "      <td>16.592</td>\n",
       "      <td>83.408</td>\n",
       "      <td>74.830</td>\n",
       "      <td>25.170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>586.000</td>\n",
       "      <td>69.000</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.396</td>\n",
       "      <td>120.903</td>\n",
       "      <td>101.458</td>\n",
       "      <td>81.686</td>\n",
       "      <td>58.163</td>\n",
       "      <td>34.815</td>\n",
       "      <td>52.899</td>\n",
       "      <td>39.691</td>\n",
       "      <td>60.309</td>\n",
       "      <td>89.466</td>\n",
       "      <td>10.534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>333.000</td>\n",
       "      <td>70.000</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.774</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.462</td>\n",
       "      <td>166.052</td>\n",
       "      <td>158.886</td>\n",
       "      <td>104.916</td>\n",
       "      <td>100.255</td>\n",
       "      <td>24.420</td>\n",
       "      <td>54.186</td>\n",
       "      <td>31.066</td>\n",
       "      <td>68.934</td>\n",
       "      <td>82.630</td>\n",
       "      <td>17.370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>197.000</td>\n",
       "      <td>67.000</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.462</td>\n",
       "      <td>155.156</td>\n",
       "      <td>120.764</td>\n",
       "      <td>74.446</td>\n",
       "      <td>73.856</td>\n",
       "      <td>15.675</td>\n",
       "      <td>52.635</td>\n",
       "      <td>22.947</td>\n",
       "      <td>77.053</td>\n",
       "      <td>74.621</td>\n",
       "      <td>25.379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>380.000</td>\n",
       "      <td>56.000</td>\n",
       "      <td>0.070</td>\n",
       "      <td>1.097</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.627</td>\n",
       "      <td>107.759</td>\n",
       "      <td>69.995</td>\n",
       "      <td>69.104</td>\n",
       "      <td>45.891</td>\n",
       "      <td>26.532</td>\n",
       "      <td>61.446</td>\n",
       "      <td>30.158</td>\n",
       "      <td>69.842</td>\n",
       "      <td>87.156</td>\n",
       "      <td>12.844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>456.000</td>\n",
       "      <td>85.000</td>\n",
       "      <td>0.068</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.495</td>\n",
       "      <td>77.189</td>\n",
       "      <td>41.573</td>\n",
       "      <td>36.823</td>\n",
       "      <td>14.536</td>\n",
       "      <td>30.855</td>\n",
       "      <td>56.496</td>\n",
       "      <td>35.323</td>\n",
       "      <td>64.677</td>\n",
       "      <td>84.288</td>\n",
       "      <td>15.712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>544.000</td>\n",
       "      <td>77.000</td>\n",
       "      <td>0.068</td>\n",
       "      <td>0.589</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.429</td>\n",
       "      <td>117.366</td>\n",
       "      <td>85.315</td>\n",
       "      <td>65.854</td>\n",
       "      <td>42.028</td>\n",
       "      <td>37.092</td>\n",
       "      <td>45.342</td>\n",
       "      <td>44.996</td>\n",
       "      <td>55.004</td>\n",
       "      <td>87.601</td>\n",
       "      <td>12.399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>374.000</td>\n",
       "      <td>84.000</td>\n",
       "      <td>0.076</td>\n",
       "      <td>0.710</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.528</td>\n",
       "      <td>30.474</td>\n",
       "      <td>22.540</td>\n",
       "      <td>3.998</td>\n",
       "      <td>2.105</td>\n",
       "      <td>28.347</td>\n",
       "      <td>59.631</td>\n",
       "      <td>32.221</td>\n",
       "      <td>67.779</td>\n",
       "      <td>81.659</td>\n",
       "      <td>18.341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  rest_cnt  gaze_cnt  rest_avg  gaze_avg  \\\n",
       "4f583872ffed48249874b00f3d389cfc   434.000    67.000     0.066     0.876   \n",
       "33d68ad55ef544fab9fd08108e3913ca   280.000    80.000     0.083     0.811   \n",
       "8598a83e2af441b8bcd0ae5d84beb875   110.000    37.000     0.089     1.336   \n",
       "37d7e67934974217830bb429bba7fd76   586.000    69.000     0.059     0.767   \n",
       "42bac596059749b5b8e8e83ae61de9b4   333.000    70.000     0.073     0.774   \n",
       "...                                    ...       ...       ...       ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94   197.000    67.000     0.080     0.786   \n",
       "611d35a7c3e940cc82495e53c2a8532d   380.000    56.000     0.070     1.097   \n",
       "3b6fda285d9e412eb081986b2f22a2e3   456.000    85.000     0.068     0.665   \n",
       "18ffb0abdc8642098c479380bfa533d1   544.000    77.000     0.068     0.589   \n",
       "c06f123b35b74bb489ec239b1cac9eb4   374.000    84.000     0.076     0.710   \n",
       "\n",
       "                                  rest_mdn  gaze_mdn  rest_dist_avg  \\\n",
       "4f583872ffed48249874b00f3d389cfc     0.033     0.495        104.681   \n",
       "33d68ad55ef544fab9fd08108e3913ca     0.066     0.611         20.270   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     0.066     0.660         95.261   \n",
       "37d7e67934974217830bb429bba7fd76     0.033     0.396        120.903   \n",
       "42bac596059749b5b8e8e83ae61de9b4     0.066     0.462        166.052   \n",
       "...                                    ...       ...            ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94     0.066     0.462        155.156   \n",
       "611d35a7c3e940cc82495e53c2a8532d     0.033     0.627        107.759   \n",
       "3b6fda285d9e412eb081986b2f22a2e3     0.033     0.495         77.189   \n",
       "18ffb0abdc8642098c479380bfa533d1     0.033     0.429        117.366   \n",
       "c06f123b35b74bb489ec239b1cac9eb4     0.066     0.528         30.474   \n",
       "\n",
       "                                  gaze_dist_avg  rest_dist_mdn  gaze_dist_mdn  \\\n",
       "4f583872ffed48249874b00f3d389cfc        100.493         85.326         66.196   \n",
       "33d68ad55ef544fab9fd08108e3913ca         10.064          2.389          2.369   \n",
       "8598a83e2af441b8bcd0ae5d84beb875        148.664         47.726         92.824   \n",
       "37d7e67934974217830bb429bba7fd76        101.458         81.686         58.163   \n",
       "42bac596059749b5b8e8e83ae61de9b4        158.886        104.916        100.255   \n",
       "...                                         ...            ...            ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94        120.764         74.446         73.856   \n",
       "611d35a7c3e940cc82495e53c2a8532d         69.995         69.104         45.891   \n",
       "3b6fda285d9e412eb081986b2f22a2e3         41.573         36.823         14.536   \n",
       "18ffb0abdc8642098c479380bfa533d1         85.315         65.854         42.028   \n",
       "c06f123b35b74bb489ec239b1cac9eb4         22.540          3.998          2.105   \n",
       "\n",
       "                                  rest_len  gaze_len  rest_l_%  gaze_l_%  \\\n",
       "4f583872ffed48249874b00f3d389cfc    28.644    58.707    32.792    67.208   \n",
       "33d68ad55ef544fab9fd08108e3913ca    23.232    64.911    26.357    73.643   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     9.834    49.434    16.592    83.408   \n",
       "37d7e67934974217830bb429bba7fd76    34.815    52.899    39.691    60.309   \n",
       "42bac596059749b5b8e8e83ae61de9b4    24.420    54.186    31.066    68.934   \n",
       "...                                    ...       ...       ...       ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94    15.675    52.635    22.947    77.053   \n",
       "611d35a7c3e940cc82495e53c2a8532d    26.532    61.446    30.158    69.842   \n",
       "3b6fda285d9e412eb081986b2f22a2e3    30.855    56.496    35.323    64.677   \n",
       "18ffb0abdc8642098c479380bfa533d1    37.092    45.342    44.996    55.004   \n",
       "c06f123b35b74bb489ec239b1cac9eb4    28.347    59.631    32.221    67.779   \n",
       "\n",
       "                                  rest_c_%  gaze_c_%  \n",
       "4f583872ffed48249874b00f3d389cfc    86.627    13.373  \n",
       "33d68ad55ef544fab9fd08108e3913ca    77.778    22.222  \n",
       "8598a83e2af441b8bcd0ae5d84beb875    74.830    25.170  \n",
       "37d7e67934974217830bb429bba7fd76    89.466    10.534  \n",
       "42bac596059749b5b8e8e83ae61de9b4    82.630    17.370  \n",
       "...                                    ...       ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94    74.621    25.379  \n",
       "611d35a7c3e940cc82495e53c2a8532d    87.156    12.844  \n",
       "3b6fda285d9e412eb081986b2f22a2e3    84.288    15.712  \n",
       "18ffb0abdc8642098c479380bfa533d1    87.601    12.399  \n",
       "c06f123b35b74bb489ec239b1cac9eb4    81.659    18.341  \n",
       "\n",
       "[24356 rows x 16 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buildings</th>\n",
       "      <th>Dynamic Cars</th>\n",
       "      <th>Dynamic Pedestrians</th>\n",
       "      <th>Inside Car</th>\n",
       "      <th>Nature</th>\n",
       "      <th>Roads</th>\n",
       "      <th>Static Cars</th>\n",
       "      <th>Static Pedestrians</th>\n",
       "      <th>Street Objects</th>\n",
       "      <th>Traffic Lights</th>\n",
       "      <th>Traffic Signs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>31.366</td>\n",
       "      <td>1.574</td>\n",
       "      <td>1.911</td>\n",
       "      <td>3.204</td>\n",
       "      <td>6.127</td>\n",
       "      <td>31.141</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.967</td>\n",
       "      <td>22.709</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.000</td>\n",
       "      <td>44.179</td>\n",
       "      <td>0.458</td>\n",
       "      <td>46.772</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.559</td>\n",
       "      <td>7.117</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>77.503</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.268</td>\n",
       "      <td>1.869</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.734</td>\n",
       "      <td>18.625</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>32.626</td>\n",
       "      <td>4.741</td>\n",
       "      <td>2.121</td>\n",
       "      <td>8.921</td>\n",
       "      <td>9.545</td>\n",
       "      <td>2.121</td>\n",
       "      <td>0.624</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.805</td>\n",
       "      <td>35.496</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>63.825</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.974</td>\n",
       "      <td>11.815</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4.263</td>\n",
       "      <td>19.123</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>64.702</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>12.226</td>\n",
       "      <td>5.204</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.944</td>\n",
       "      <td>15.925</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>30.988</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>39.527</td>\n",
       "      <td>7.626</td>\n",
       "      <td>4.619</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.719</td>\n",
       "      <td>14.930</td>\n",
       "      <td>0.591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>13.843</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.818</td>\n",
       "      <td>32.535</td>\n",
       "      <td>1.168</td>\n",
       "      <td>29.381</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.986</td>\n",
       "      <td>10.397</td>\n",
       "      <td>9.871</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>32.460</td>\n",
       "      <td>0.582</td>\n",
       "      <td>0.000</td>\n",
       "      <td>12.082</td>\n",
       "      <td>10.917</td>\n",
       "      <td>14.119</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.275</td>\n",
       "      <td>25.182</td>\n",
       "      <td>1.383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>0.719</td>\n",
       "      <td>4.261</td>\n",
       "      <td>0.000</td>\n",
       "      <td>58.882</td>\n",
       "      <td>2.490</td>\n",
       "      <td>21.251</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.210</td>\n",
       "      <td>6.198</td>\n",
       "      <td>2.988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Buildings  Dynamic Cars  \\\n",
       "4f583872ffed48249874b00f3d389cfc     31.366         1.574   \n",
       "33d68ad55ef544fab9fd08108e3913ca      0.000         0.915   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     77.503         0.000   \n",
       "37d7e67934974217830bb429bba7fd76     32.626         4.741   \n",
       "42bac596059749b5b8e8e83ae61de9b4     63.825         0.000   \n",
       "...                                     ...           ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94     64.702         0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d     30.988         0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3     13.843         0.000   \n",
       "18ffb0abdc8642098c479380bfa533d1     32.460         0.582   \n",
       "c06f123b35b74bb489ec239b1cac9eb4      0.719         4.261   \n",
       "\n",
       "                                  Dynamic Pedestrians  Inside Car  Nature  \\\n",
       "4f583872ffed48249874b00f3d389cfc                1.911       3.204   6.127   \n",
       "33d68ad55ef544fab9fd08108e3913ca                0.000      44.179   0.458   \n",
       "8598a83e2af441b8bcd0ae5d84beb875                0.000       1.268   1.869   \n",
       "37d7e67934974217830bb429bba7fd76                2.121       8.921   9.545   \n",
       "42bac596059749b5b8e8e83ae61de9b4                0.000       0.974  11.815   \n",
       "...                                               ...         ...     ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94                0.000      12.226   5.204   \n",
       "611d35a7c3e940cc82495e53c2a8532d                0.000      39.527   7.626   \n",
       "3b6fda285d9e412eb081986b2f22a2e3                0.818      32.535   1.168   \n",
       "18ffb0abdc8642098c479380bfa533d1                0.000      12.082  10.917   \n",
       "c06f123b35b74bb489ec239b1cac9eb4                0.000      58.882   2.490   \n",
       "\n",
       "                                  Roads  Static Cars  Static Pedestrians  \\\n",
       "4f583872ffed48249874b00f3d389cfc 31.141        0.000               0.000   \n",
       "33d68ad55ef544fab9fd08108e3913ca 46.772        0.000               0.000   \n",
       "8598a83e2af441b8bcd0ae5d84beb875  0.000        0.000               0.000   \n",
       "37d7e67934974217830bb429bba7fd76  2.121        0.624               0.000   \n",
       "42bac596059749b5b8e8e83ae61de9b4  0.000        0.000               0.000   \n",
       "...                                 ...          ...                 ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94  0.000        0.000               0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d  4.619        0.000               0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3 29.381        0.000               1.986   \n",
       "18ffb0abdc8642098c479380bfa533d1 14.119        0.000               0.000   \n",
       "c06f123b35b74bb489ec239b1cac9eb4 21.251        0.000               0.000   \n",
       "\n",
       "                                  Street Objects  Traffic Lights  \\\n",
       "4f583872ffed48249874b00f3d389cfc           1.967          22.709   \n",
       "33d68ad55ef544fab9fd08108e3913ca           0.559           7.117   \n",
       "8598a83e2af441b8bcd0ae5d84beb875           0.734          18.625   \n",
       "37d7e67934974217830bb429bba7fd76           3.805          35.496   \n",
       "42bac596059749b5b8e8e83ae61de9b4           4.263          19.123   \n",
       "...                                          ...             ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94           1.944          15.925   \n",
       "611d35a7c3e940cc82495e53c2a8532d           1.719          14.930   \n",
       "3b6fda285d9e412eb081986b2f22a2e3          10.397           9.871   \n",
       "18ffb0abdc8642098c479380bfa533d1           3.275          25.182   \n",
       "c06f123b35b74bb489ec239b1cac9eb4           3.210           6.198   \n",
       "\n",
       "                                  Traffic Signs  \n",
       "4f583872ffed48249874b00f3d389cfc          0.000  \n",
       "33d68ad55ef544fab9fd08108e3913ca          0.000  \n",
       "8598a83e2af441b8bcd0ae5d84beb875          0.000  \n",
       "37d7e67934974217830bb429bba7fd76          0.000  \n",
       "42bac596059749b5b8e8e83ae61de9b4          0.000  \n",
       "...                                         ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94          0.000  \n",
       "611d35a7c3e940cc82495e53c2a8532d          0.591  \n",
       "3b6fda285d9e412eb081986b2f22a2e3          0.000  \n",
       "18ffb0abdc8642098c479380bfa533d1          1.383  \n",
       "c06f123b35b74bb489ec239b1cac9eb4          2.988  \n",
       "\n",
       "[24356 rows x 11 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buildings</th>\n",
       "      <th>Dynamic Cars</th>\n",
       "      <th>Dynamic Pedestrians</th>\n",
       "      <th>Inside Car</th>\n",
       "      <th>Nature</th>\n",
       "      <th>Roads</th>\n",
       "      <th>Static Cars</th>\n",
       "      <th>Static Pedestrians</th>\n",
       "      <th>Street Objects</th>\n",
       "      <th>Traffic Lights</th>\n",
       "      <th>Traffic Signs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>34.328</td>\n",
       "      <td>2.985</td>\n",
       "      <td>1.493</td>\n",
       "      <td>4.478</td>\n",
       "      <td>11.940</td>\n",
       "      <td>28.358</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.970</td>\n",
       "      <td>10.448</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>0.000</td>\n",
       "      <td>2.500</td>\n",
       "      <td>0.000</td>\n",
       "      <td>52.500</td>\n",
       "      <td>1.250</td>\n",
       "      <td>40.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.250</td>\n",
       "      <td>2.500</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>64.865</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.405</td>\n",
       "      <td>8.108</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.703</td>\n",
       "      <td>18.919</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>37.681</td>\n",
       "      <td>4.348</td>\n",
       "      <td>1.449</td>\n",
       "      <td>14.493</td>\n",
       "      <td>15.942</td>\n",
       "      <td>5.797</td>\n",
       "      <td>1.449</td>\n",
       "      <td>0.000</td>\n",
       "      <td>8.696</td>\n",
       "      <td>10.145</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>57.143</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.857</td>\n",
       "      <td>20.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>7.143</td>\n",
       "      <td>12.857</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>62.687</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>13.433</td>\n",
       "      <td>8.955</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.985</td>\n",
       "      <td>11.940</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>33.929</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>32.143</td>\n",
       "      <td>14.286</td>\n",
       "      <td>7.143</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.357</td>\n",
       "      <td>5.357</td>\n",
       "      <td>1.786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>11.765</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.176</td>\n",
       "      <td>30.588</td>\n",
       "      <td>2.353</td>\n",
       "      <td>34.118</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.353</td>\n",
       "      <td>10.588</td>\n",
       "      <td>7.059</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>31.169</td>\n",
       "      <td>1.299</td>\n",
       "      <td>0.000</td>\n",
       "      <td>16.883</td>\n",
       "      <td>15.584</td>\n",
       "      <td>12.987</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.195</td>\n",
       "      <td>14.286</td>\n",
       "      <td>2.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>1.190</td>\n",
       "      <td>3.571</td>\n",
       "      <td>0.000</td>\n",
       "      <td>57.143</td>\n",
       "      <td>4.762</td>\n",
       "      <td>23.810</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.381</td>\n",
       "      <td>2.381</td>\n",
       "      <td>4.762</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Buildings  Dynamic Cars  \\\n",
       "4f583872ffed48249874b00f3d389cfc     34.328         2.985   \n",
       "33d68ad55ef544fab9fd08108e3913ca      0.000         2.500   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     64.865         0.000   \n",
       "37d7e67934974217830bb429bba7fd76     37.681         4.348   \n",
       "42bac596059749b5b8e8e83ae61de9b4     57.143         0.000   \n",
       "...                                     ...           ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94     62.687         0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d     33.929         0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3     11.765         0.000   \n",
       "18ffb0abdc8642098c479380bfa533d1     31.169         1.299   \n",
       "c06f123b35b74bb489ec239b1cac9eb4      1.190         3.571   \n",
       "\n",
       "                                  Dynamic Pedestrians  Inside Car  Nature  \\\n",
       "4f583872ffed48249874b00f3d389cfc                1.493       4.478  11.940   \n",
       "33d68ad55ef544fab9fd08108e3913ca                0.000      52.500   1.250   \n",
       "8598a83e2af441b8bcd0ae5d84beb875                0.000       5.405   8.108   \n",
       "37d7e67934974217830bb429bba7fd76                1.449      14.493  15.942   \n",
       "42bac596059749b5b8e8e83ae61de9b4                0.000       2.857  20.000   \n",
       "...                                               ...         ...     ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94                0.000      13.433   8.955   \n",
       "611d35a7c3e940cc82495e53c2a8532d                0.000      32.143  14.286   \n",
       "3b6fda285d9e412eb081986b2f22a2e3                1.176      30.588   2.353   \n",
       "18ffb0abdc8642098c479380bfa533d1                0.000      16.883  15.584   \n",
       "c06f123b35b74bb489ec239b1cac9eb4                0.000      57.143   4.762   \n",
       "\n",
       "                                  Roads  Static Cars  Static Pedestrians  \\\n",
       "4f583872ffed48249874b00f3d389cfc 28.358        0.000               0.000   \n",
       "33d68ad55ef544fab9fd08108e3913ca 40.000        0.000               0.000   \n",
       "8598a83e2af441b8bcd0ae5d84beb875  0.000        0.000               0.000   \n",
       "37d7e67934974217830bb429bba7fd76  5.797        1.449               0.000   \n",
       "42bac596059749b5b8e8e83ae61de9b4  0.000        0.000               0.000   \n",
       "...                                 ...          ...                 ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94  0.000        0.000               0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d  7.143        0.000               0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3 34.118        0.000               2.353   \n",
       "18ffb0abdc8642098c479380bfa533d1 12.987        0.000               0.000   \n",
       "c06f123b35b74bb489ec239b1cac9eb4 23.810        0.000               0.000   \n",
       "\n",
       "                                  Street Objects  Traffic Lights  \\\n",
       "4f583872ffed48249874b00f3d389cfc           5.970          10.448   \n",
       "33d68ad55ef544fab9fd08108e3913ca           1.250           2.500   \n",
       "8598a83e2af441b8bcd0ae5d84beb875           2.703          18.919   \n",
       "37d7e67934974217830bb429bba7fd76           8.696          10.145   \n",
       "42bac596059749b5b8e8e83ae61de9b4           7.143          12.857   \n",
       "...                                          ...             ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94           2.985          11.940   \n",
       "611d35a7c3e940cc82495e53c2a8532d           5.357           5.357   \n",
       "3b6fda285d9e412eb081986b2f22a2e3          10.588           7.059   \n",
       "18ffb0abdc8642098c479380bfa533d1           5.195          14.286   \n",
       "c06f123b35b74bb489ec239b1cac9eb4           2.381           2.381   \n",
       "\n",
       "                                  Traffic Signs  \n",
       "4f583872ffed48249874b00f3d389cfc          0.000  \n",
       "33d68ad55ef544fab9fd08108e3913ca          0.000  \n",
       "8598a83e2af441b8bcd0ae5d84beb875          0.000  \n",
       "37d7e67934974217830bb429bba7fd76          0.000  \n",
       "42bac596059749b5b8e8e83ae61de9b4          0.000  \n",
       "...                                         ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94          0.000  \n",
       "611d35a7c3e940cc82495e53c2a8532d          1.786  \n",
       "3b6fda285d9e412eb081986b2f22a2e3          0.000  \n",
       "18ffb0abdc8642098c479380bfa533d1          2.597  \n",
       "c06f123b35b74bb489ec239b1cac9eb4          4.762  \n",
       "\n",
       "[24356 rows x 11 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buildings</th>\n",
       "      <th>Dynamic Cars</th>\n",
       "      <th>Dynamic Pedestrians</th>\n",
       "      <th>Inside Car</th>\n",
       "      <th>Nature</th>\n",
       "      <th>Roads</th>\n",
       "      <th>Static Cars</th>\n",
       "      <th>Static Pedestrians</th>\n",
       "      <th>Street Objects</th>\n",
       "      <th>Traffic Lights</th>\n",
       "      <th>Traffic Signs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>18.414</td>\n",
       "      <td>0.924</td>\n",
       "      <td>1.122</td>\n",
       "      <td>1.881</td>\n",
       "      <td>3.597</td>\n",
       "      <td>18.282</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.155</td>\n",
       "      <td>13.332</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.594</td>\n",
       "      <td>0.000</td>\n",
       "      <td>28.677</td>\n",
       "      <td>0.297</td>\n",
       "      <td>30.360</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.363</td>\n",
       "      <td>4.620</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>38.313</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.627</td>\n",
       "      <td>0.924</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.363</td>\n",
       "      <td>9.207</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>17.259</td>\n",
       "      <td>2.508</td>\n",
       "      <td>1.122</td>\n",
       "      <td>4.719</td>\n",
       "      <td>5.049</td>\n",
       "      <td>1.122</td>\n",
       "      <td>0.330</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.013</td>\n",
       "      <td>18.777</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>34.584</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.528</td>\n",
       "      <td>6.402</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.310</td>\n",
       "      <td>10.362</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>34.056</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>6.435</td>\n",
       "      <td>2.739</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.023</td>\n",
       "      <td>8.382</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>19.041</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>24.288</td>\n",
       "      <td>4.686</td>\n",
       "      <td>2.838</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.056</td>\n",
       "      <td>9.174</td>\n",
       "      <td>0.363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>7.821</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.462</td>\n",
       "      <td>18.381</td>\n",
       "      <td>0.660</td>\n",
       "      <td>16.599</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.122</td>\n",
       "      <td>5.874</td>\n",
       "      <td>5.577</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>14.718</td>\n",
       "      <td>0.264</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.478</td>\n",
       "      <td>4.950</td>\n",
       "      <td>6.402</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.485</td>\n",
       "      <td>11.418</td>\n",
       "      <td>0.627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>0.429</td>\n",
       "      <td>2.541</td>\n",
       "      <td>0.000</td>\n",
       "      <td>35.112</td>\n",
       "      <td>1.485</td>\n",
       "      <td>12.672</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.914</td>\n",
       "      <td>3.696</td>\n",
       "      <td>1.782</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Buildings  Dynamic Cars  \\\n",
       "4f583872ffed48249874b00f3d389cfc     18.414         0.924   \n",
       "33d68ad55ef544fab9fd08108e3913ca      0.000         0.594   \n",
       "8598a83e2af441b8bcd0ae5d84beb875     38.313         0.000   \n",
       "37d7e67934974217830bb429bba7fd76     17.259         2.508   \n",
       "42bac596059749b5b8e8e83ae61de9b4     34.584         0.000   \n",
       "...                                     ...           ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94     34.056         0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d     19.041         0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3      7.821         0.000   \n",
       "18ffb0abdc8642098c479380bfa533d1     14.718         0.264   \n",
       "c06f123b35b74bb489ec239b1cac9eb4      0.429         2.541   \n",
       "\n",
       "                                  Dynamic Pedestrians  Inside Car  Nature  \\\n",
       "4f583872ffed48249874b00f3d389cfc                1.122       1.881   3.597   \n",
       "33d68ad55ef544fab9fd08108e3913ca                0.000      28.677   0.297   \n",
       "8598a83e2af441b8bcd0ae5d84beb875                0.000       0.627   0.924   \n",
       "37d7e67934974217830bb429bba7fd76                1.122       4.719   5.049   \n",
       "42bac596059749b5b8e8e83ae61de9b4                0.000       0.528   6.402   \n",
       "...                                               ...         ...     ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94                0.000       6.435   2.739   \n",
       "611d35a7c3e940cc82495e53c2a8532d                0.000      24.288   4.686   \n",
       "3b6fda285d9e412eb081986b2f22a2e3                0.462      18.381   0.660   \n",
       "18ffb0abdc8642098c479380bfa533d1                0.000       5.478   4.950   \n",
       "c06f123b35b74bb489ec239b1cac9eb4                0.000      35.112   1.485   \n",
       "\n",
       "                                  Roads  Static Cars  Static Pedestrians  \\\n",
       "4f583872ffed48249874b00f3d389cfc 18.282        0.000               0.000   \n",
       "33d68ad55ef544fab9fd08108e3913ca 30.360        0.000               0.000   \n",
       "8598a83e2af441b8bcd0ae5d84beb875  0.000        0.000               0.000   \n",
       "37d7e67934974217830bb429bba7fd76  1.122        0.330               0.000   \n",
       "42bac596059749b5b8e8e83ae61de9b4  0.000        0.000               0.000   \n",
       "...                                 ...          ...                 ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94  0.000        0.000               0.000   \n",
       "611d35a7c3e940cc82495e53c2a8532d  2.838        0.000               0.000   \n",
       "3b6fda285d9e412eb081986b2f22a2e3 16.599        0.000               1.122   \n",
       "18ffb0abdc8642098c479380bfa533d1  6.402        0.000               0.000   \n",
       "c06f123b35b74bb489ec239b1cac9eb4 12.672        0.000               0.000   \n",
       "\n",
       "                                  Street Objects  Traffic Lights  \\\n",
       "4f583872ffed48249874b00f3d389cfc           1.155          13.332   \n",
       "33d68ad55ef544fab9fd08108e3913ca           0.363           4.620   \n",
       "8598a83e2af441b8bcd0ae5d84beb875           0.363           9.207   \n",
       "37d7e67934974217830bb429bba7fd76           2.013          18.777   \n",
       "42bac596059749b5b8e8e83ae61de9b4           2.310          10.362   \n",
       "...                                          ...             ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94           1.023           8.382   \n",
       "611d35a7c3e940cc82495e53c2a8532d           1.056           9.174   \n",
       "3b6fda285d9e412eb081986b2f22a2e3           5.874           5.577   \n",
       "18ffb0abdc8642098c479380bfa533d1           1.485          11.418   \n",
       "c06f123b35b74bb489ec239b1cac9eb4           1.914           3.696   \n",
       "\n",
       "                                  Traffic Signs  \n",
       "4f583872ffed48249874b00f3d389cfc          0.000  \n",
       "33d68ad55ef544fab9fd08108e3913ca          0.000  \n",
       "8598a83e2af441b8bcd0ae5d84beb875          0.000  \n",
       "37d7e67934974217830bb429bba7fd76          0.000  \n",
       "42bac596059749b5b8e8e83ae61de9b4          0.000  \n",
       "...                                         ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94          0.000  \n",
       "611d35a7c3e940cc82495e53c2a8532d          0.363  \n",
       "3b6fda285d9e412eb081986b2f22a2e3          0.000  \n",
       "18ffb0abdc8642098c479380bfa533d1          0.627  \n",
       "c06f123b35b74bb489ec239b1cac9eb4          1.782  \n",
       "\n",
       "[24356 rows x 11 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buildings</th>\n",
       "      <th>Dynamic Cars</th>\n",
       "      <th>Dynamic Pedestrians</th>\n",
       "      <th>Inside Car</th>\n",
       "      <th>Nature</th>\n",
       "      <th>Roads</th>\n",
       "      <th>Static Cars</th>\n",
       "      <th>Static Pedestrians</th>\n",
       "      <th>Street Objects</th>\n",
       "      <th>Traffic Lights</th>\n",
       "      <th>Traffic Signs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Buildings  Dynamic Cars  \\\n",
       "4f583872ffed48249874b00f3d389cfc         23             2   \n",
       "33d68ad55ef544fab9fd08108e3913ca          0             2   \n",
       "8598a83e2af441b8bcd0ae5d84beb875         24             0   \n",
       "37d7e67934974217830bb429bba7fd76         26             3   \n",
       "42bac596059749b5b8e8e83ae61de9b4         40             0   \n",
       "...                                     ...           ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94         42             0   \n",
       "611d35a7c3e940cc82495e53c2a8532d         19             0   \n",
       "3b6fda285d9e412eb081986b2f22a2e3         10             0   \n",
       "18ffb0abdc8642098c479380bfa533d1         24             1   \n",
       "c06f123b35b74bb489ec239b1cac9eb4          1             3   \n",
       "\n",
       "                                  Dynamic Pedestrians  Inside Car  Nature  \\\n",
       "4f583872ffed48249874b00f3d389cfc                    1           3       8   \n",
       "33d68ad55ef544fab9fd08108e3913ca                    0          42       1   \n",
       "8598a83e2af441b8bcd0ae5d84beb875                    0           2       3   \n",
       "37d7e67934974217830bb429bba7fd76                    1          10      11   \n",
       "42bac596059749b5b8e8e83ae61de9b4                    0           2      14   \n",
       "...                                               ...         ...     ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94                    0           9       6   \n",
       "611d35a7c3e940cc82495e53c2a8532d                    0          18       8   \n",
       "3b6fda285d9e412eb081986b2f22a2e3                    1          26       2   \n",
       "18ffb0abdc8642098c479380bfa533d1                    0          13      12   \n",
       "c06f123b35b74bb489ec239b1cac9eb4                    0          48       4   \n",
       "\n",
       "                                  Roads  Static Cars  Static Pedestrians  \\\n",
       "4f583872ffed48249874b00f3d389cfc     19            0                   0   \n",
       "33d68ad55ef544fab9fd08108e3913ca     32            0                   0   \n",
       "8598a83e2af441b8bcd0ae5d84beb875      0            0                   0   \n",
       "37d7e67934974217830bb429bba7fd76      4            1                   0   \n",
       "42bac596059749b5b8e8e83ae61de9b4      0            0                   0   \n",
       "...                                 ...          ...                 ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94      0            0                   0   \n",
       "611d35a7c3e940cc82495e53c2a8532d      4            0                   0   \n",
       "3b6fda285d9e412eb081986b2f22a2e3     29            0                   2   \n",
       "18ffb0abdc8642098c479380bfa533d1     10            0                   0   \n",
       "c06f123b35b74bb489ec239b1cac9eb4     20            0                   0   \n",
       "\n",
       "                                  Street Objects  Traffic Lights  \\\n",
       "4f583872ffed48249874b00f3d389cfc               4               7   \n",
       "33d68ad55ef544fab9fd08108e3913ca               1               2   \n",
       "8598a83e2af441b8bcd0ae5d84beb875               1               7   \n",
       "37d7e67934974217830bb429bba7fd76               6               7   \n",
       "42bac596059749b5b8e8e83ae61de9b4               5               9   \n",
       "...                                          ...             ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94               2               8   \n",
       "611d35a7c3e940cc82495e53c2a8532d               3               3   \n",
       "3b6fda285d9e412eb081986b2f22a2e3               9               6   \n",
       "18ffb0abdc8642098c479380bfa533d1               4              11   \n",
       "c06f123b35b74bb489ec239b1cac9eb4               2               2   \n",
       "\n",
       "                                  Traffic Signs  \n",
       "4f583872ffed48249874b00f3d389cfc              0  \n",
       "33d68ad55ef544fab9fd08108e3913ca              0  \n",
       "8598a83e2af441b8bcd0ae5d84beb875              0  \n",
       "37d7e67934974217830bb429bba7fd76              0  \n",
       "42bac596059749b5b8e8e83ae61de9b4              0  \n",
       "...                                         ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94              0  \n",
       "611d35a7c3e940cc82495e53c2a8532d              1  \n",
       "3b6fda285d9e412eb081986b2f22a2e3              0  \n",
       "18ffb0abdc8642098c479380bfa533d1              2  \n",
       "c06f123b35b74bb489ec239b1cac9eb4              4  \n",
       "\n",
       "[24356 rows x 11 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def gazes_vs_rest(uid):\n",
    "    \"\"\"\n",
    "    Compute gaze vs rest stats and store gaze length and gaze counts (events)\n",
    "    of a participant given its UID.\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant unique identifier.\n",
    "    \"\"\"\n",
    "    h_events = pd.read_csv(f\"{PATH_PROC}/hit_events/{uid}.csv\")\n",
    "    # filter out Sky events since they are uninformative of gaze/rest\n",
    "    h_events = h_events[h_events.group != \"Sky\"]\n",
    "    total = h_events.index.size\n",
    "\n",
    "    # disentangle gazes and rest events\n",
    "    rest = h_events[h_events[\"type\"] == \"rest\"]\n",
    "    gazes = h_events[h_events[\"type\"] == \"gaze\"]\n",
    "\n",
    "    # compute length (time in s) and event counts\n",
    "    total_r = rest.index.size\n",
    "    total_g = gazes.index.size\n",
    "    rest_len = rest[\"length\"].sum()\n",
    "    gaze_len = gazes[\"length\"].sum()\n",
    "    total_len = rest_len + gaze_len\n",
    "\n",
    "    # store stats\n",
    "    hev_cnts[uid][\"rest_cnt\"] = total_r\n",
    "    hev_cnts[uid][\"gaze_cnt\"] = total_g\n",
    "    hev_cnts[uid][\"rest_avg\"] = rest[\"length\"].mean()\n",
    "    hev_cnts[uid][\"gaze_avg\"] = gazes[\"length\"].mean()\n",
    "    hev_cnts[uid][\"rest_mdn\"] = rest[\"length\"].median()\n",
    "    hev_cnts[uid][\"gaze_mdn\"] = gazes[\"length\"].median()\n",
    "    hev_cnts[uid][\"rest_dist_avg\"] = rest[\"distance\"].mean()\n",
    "    hev_cnts[uid][\"gaze_dist_avg\"] = gazes[\"distance\"].mean()\n",
    "    hev_cnts[uid][\"rest_dist_mdn\"] = rest[\"distance\"].median()\n",
    "    hev_cnts[uid][\"gaze_dist_mdn\"] = gazes[\"distance\"].median()\n",
    "    hev_cnts[uid][\"rest_len\"] = rest_len\n",
    "    hev_cnts[uid][\"gaze_len\"] = gaze_len\n",
    "    hev_cnts[uid][\"rest_l_%\"] = rest_len * 100 / total_len\n",
    "    hev_cnts[uid][\"gaze_l_%\"] = gaze_len * 100 / total_len\n",
    "    hev_cnts[uid][\"rest_c_%\"] = total_r * 100 / total\n",
    "    hev_cnts[uid][\"gaze_c_%\"] = total_g * 100 / total\n",
    "\n",
    "    # iterate over objects groups for current participant\n",
    "    grps = gazes.group.unique()\n",
    "    for grp in grps:\n",
    "        # gazes for the current group, compute and store lengths and counts\n",
    "        sel = gazes[gazes.group == grp]\n",
    "        g_len = sel[\"length\"].sum()\n",
    "        g_cnt = sel.index.size\n",
    "        grp_lpercs[uid][grp] = g_len * 100 / gaze_len\n",
    "        grp_cpercs[uid][grp] = g_cnt * 100 / total_g\n",
    "        grp_len[uid][grp] = g_len\n",
    "        grp_cnt[uid][grp] = g_cnt\n",
    "    # iterate over all groups possible\n",
    "    for grp in grp_cats:\n",
    "        # if the group is not current participant, add 0 counter\n",
    "        if not grp_lpercs[uid][grp]:\n",
    "            grp_lpercs[uid][grp] = 0\n",
    "            grp_cpercs[uid][grp] = 0\n",
    "            grp_len[uid][grp] = 0\n",
    "            grp_cnt[uid][grp] = 0\n",
    "\n",
    "\n",
    "# participants ids\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "cats = [\n",
    "    \"rest_cnt\",\n",
    "    \"gaze_cnt\",\n",
    "    \"rest_avg\",\n",
    "    \"gaze_avg\",\n",
    "    \"rest_mdn\",\n",
    "    \"gaze_mdn\",\n",
    "    \"rest_dist_avg\",\n",
    "    \"gaze_dist_avg\",\n",
    "    \"rest_dist_mdn\",\n",
    "    \"gaze_dist_mdn\",\n",
    "    \"rest_len\",\n",
    "    \"gaze_len\",\n",
    "    \"rest_l_%\",\n",
    "    \"gaze_l_%\",\n",
    "    \"rest_c_%\",\n",
    "    \"gaze_c_%\",\n",
    "]\n",
    "stats = {cat: None for cat in cats}\n",
    "\n",
    "grp_cats = [\n",
    "    \"Buildings\",\n",
    "    \"Dynamic Cars\",\n",
    "    \"Dynamic Pedestrians\",\n",
    "    \"Inside Car\",\n",
    "    \"Nature\",\n",
    "    \"Roads\",\n",
    "    # \"Sky\",\n",
    "    \"Static Cars\",\n",
    "    \"Static Pedestrians\",\n",
    "    \"Street Objects\",\n",
    "    \"Traffic Lights\",\n",
    "    \"Traffic Signs\",\n",
    "]\n",
    "grp_stats = {cat: None for cat in grp_cats}\n",
    "\n",
    "manager = Manager()  # manage shared memory types\n",
    "\n",
    "# convert into memory-shared dict of dicts for multiprocessing\n",
    "hev_cnts = manager.dict({uid: manager.dict(stats) for uid in uids})\n",
    "grp_lpercs = manager.dict({uid: manager.dict(grp_stats) for uid in uids})\n",
    "grp_cpercs = manager.dict({uid: manager.dict(grp_stats) for uid in uids})\n",
    "grp_len = manager.dict({uid: manager.dict(grp_stats) for uid in uids})\n",
    "grp_cnt = manager.dict({uid: manager.dict(grp_stats) for uid in uids})\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap_unordered(func=gazes_vs_rest, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()\n",
    "\n",
    "# parse and store collected data\n",
    "hev_cnts = dict(hev_cnts)\n",
    "hev_cnts = {k: dict(v) for k, v in hev_cnts.items()}\n",
    "hev_cnts = pd.DataFrame(hev_cnts).transpose()\n",
    "hev_cnts.to_csv(\"./hit_events_counters.csv\")\n",
    "display(hev_cnts)\n",
    "\n",
    "grp_lpercs = dict(grp_lpercs)\n",
    "grp_cpercs = dict(grp_cpercs)\n",
    "grp_len = dict(grp_len)\n",
    "grp_cnt = dict(grp_cnt)\n",
    "grp_lpercs = {k: dict(v) for k, v in grp_lpercs.items()}\n",
    "grp_cpercs = {k: dict(v) for k, v in grp_cpercs.items()}\n",
    "grp_len = {k: dict(v) for k, v in grp_len.items()}\n",
    "grp_cnt = {k: dict(v) for k, v in grp_cnt.items()}\n",
    "grp_lpercs = pd.DataFrame(grp_lpercs).transpose()\n",
    "grp_cpercs = pd.DataFrame(grp_cpercs).transpose()\n",
    "grp_len = pd.DataFrame(grp_len).transpose()\n",
    "grp_cnt = pd.DataFrame(grp_cnt).transpose()\n",
    "grp_lpercs.to_csv(\"./group_length_percentages.csv\")\n",
    "grp_cpercs.to_csv(\"./group_counts_percentages.csv\")\n",
    "grp_len.to_csv(\"./group_lengths.csv\")\n",
    "grp_cnt.to_csv(\"./group_counts.csv\")\n",
    "display(grp_lpercs)\n",
    "display(grp_cpercs)\n",
    "display(grp_len)\n",
    "display(grp_cnt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (msw-bmbf_da)",
   "language": "python",
   "name": "msw-bmbf_da"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}