1_data_parsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Dependencies__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy  # copy big/deep objects by value\n",
    "import datetime  # datetime operations\n",
    "import itertools  # operate with iterators\n",
    "import json  # read/write from/into json format\n",
    "import os  # OS operations (read/write files/folders)\n",
    "import warnings  # hide warnings\n",
    "\n",
    "# process parallelization\n",
    "from multiprocessing import Lock, Manager, Pool, RawArray, cpu_count, managers\n",
    "\n",
    "import matplotlib.pyplot as plt  # mother of plots focr Python\n",
    "import matplotlib.ticker as ticker  # matplotlib ticker utils\n",
    "import numpy as np  # array/matrix operations (e.g. linear algebra)\n",
    "import pandas as pd  # operate with dataframes\n",
    "import seaborn as sns  # matplotlib plotting nice with shortcuts\n",
    "from IPython.display import display  # print nicely\n",
    "from tqdm.notebook import tqdm  # mother of progressbars for Python\n",
    "\n",
    "# from matplotlib.ticker import FormatStrFormatter  # tick formatter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Options and constants__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total CPU threads: 16\n"
     ]
    }
   ],
   "source": [
    "warnings.filterwarnings(\"ignore\")\n",
    "# set default float display format to 2 decimals\n",
    "pd.options.display.float_format = \"{:.3f}\".format\n",
    "\n",
    "PATH_RAYC = \"./data/raw/CsvData\"\n",
    "PATH_PROC = \"./data/processed\"\n",
    "\n",
    "# hit origin structure (parsed/new)\n",
    "HIT_ORIGIN = [\n",
    "    \"frame\",\n",
    "    \"originX\",\n",
    "    \"originY\",\n",
    "    \"originZ\",\n",
    "    \"rotationX\",\n",
    "    \"rotationY\",\n",
    "    \"rotationZ\",\n",
    "    \"rotationW\",\n",
    "    \"directionX\",\n",
    "    \"directionY\",\n",
    "    \"directionZ\",\n",
    "    \"angleX\",\n",
    "    \"angleY\",\n",
    "    \"angleZ\",\n",
    "]\n",
    "\n",
    "# hit information structure (all objects)\n",
    "HIT_INFO = [\n",
    "    \"frame\",\n",
    "    \"name\",\n",
    "    \"group\",\n",
    "    \"distance\",\n",
    "    \"objectX\",\n",
    "    \"objectY\",\n",
    "    \"objectZ\",\n",
    "    \"hitX\",\n",
    "    \"hitY\",\n",
    "    \"hitZ\",\n",
    "]\n",
    "\n",
    "GRP_REPLACE = {\n",
    "    \"ADV_linux(Clone)\": \"Inside Car\",\n",
    "    \"CarsManager\": \"Dynamic Cars\",\n",
    "    \"InsideCar\": \"Inside Car\",\n",
    "    \"Parking Cars\": \"Static Cars\",\n",
    "    \"Path_ MS Wissenschaft\": \"Roads\",\n",
    "    \"PedestriansManager\": \"Dynamic Pedestrians\",\n",
    "    \"WestRide Buildings\": \"Buildings\",\n",
    "    \"WestRide Street Prob\": \"Street Objects\",\n",
    "    \"WestRide Trees/Rocks/Nature\": \"Nature\",\n",
    "    \"Westdrive Crosswalks\": \"Crosswalks\",\n",
    "    \"Westdrive Static NPCs\": \"Static Pedestrians\",\n",
    "}\n",
    "\n",
    "# hit groups to skip\n",
    "EXCEPT_GROUPS = [\"Crosswalks\", \"End\", \"Events\", \"Start\"]\n",
    "\n",
    "# x, y, and z axis direction vectors\n",
    "X_DIR = np.array([1, 0, 0])\n",
    "Y_DIR = np.array([0, 1, 0])\n",
    "Z_DIR = np.array([0, 0, 1])\n",
    "\n",
    "# progress bar customized format\n",
    "B_FORMAT = \"\"\"📄 {n_fmt} of {total_fmt} {desc} processed: {bar} \n",
    "            {percentage:3.0f}% ⏱️{elapsed} ⏳{remaining} ⚙️{rate_fmt}{postfix}\"\"\"\n",
    "\n",
    "CORES = cpu_count()  # number of cpu threads for multiprocessing\n",
    "print(f\"Total CPU threads: {CORES}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Helper functions__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pbar_fork_hack(l=None):\n",
    "    \"\"\"\n",
    "    Hack to enforce progress bars to be displayed by fork processes on\n",
    "    IPython Apps like Jupyter Notebooks.\n",
    "\n",
    "    Avoids [IPKernelApp] WARNING | WARNING: attempted to send message from fork\n",
    "\n",
    "    Important: pass this function as argument for the initializer parameter\n",
    "    while initializing a multiprocessing pool to make it work. E.g.:\n",
    "\n",
    "    pool = Pool(processes=N_CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "    Source:\n",
    "     - https://github.com/ipython/ipython/issues/11049#issue-306086846\n",
    "     - https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308\n",
    "    \"\"\"\n",
    "    if l:\n",
    "        global lock\n",
    "        lock = l\n",
    "    print(\" \", end=\"\", flush=True)\n",
    "\n",
    "\n",
    "def angle(a, b):\n",
    "    \"\"\"\n",
    "    Angle (in degrees) between two vectors.\n",
    "\n",
    "    Given two 3D vectors (a and b):\n",
    "    a = [xa, ya, za] , b = [xb, yb, zb]\n",
    "\n",
    "    And the basic geometric formula for the dot product:\n",
    "    a · b = |a| * |b| * cos(α) => α = arccos[(a · b) / (|a| * |b|)]\n",
    "\n",
    "    In other words:\n",
    "    angle = arccos[(xa * xb + ya * yb + za * zb) / (√(xa2 + ya2 + za2) * √(xb2 + yb2 + zb2))]\n",
    "\n",
    "    Source: https://www.omnicalculator.com/math/angle-between-two-vectors\n",
    "\n",
    "    Parameters:\n",
    "        a (numpy.array): First vector\n",
    "        b (numpy.array): Second vector\n",
    "    Returns:\n",
    "        float: Angle between vectors a and b in degrees.\n",
    "    \"\"\"\n",
    "    # a · b\n",
    "    dotp = np.dot(a, b)\n",
    "\n",
    "    # magnitude vectors |a| and |b|\n",
    "    m_a = np.linalg.norm(a)\n",
    "    m_b = np.linalg.norm(b)\n",
    "\n",
    "    # |a| * |b|\n",
    "    m_prod = m_a * m_b\n",
    "\n",
    "    # (a * b) / (|a| * |b|)\n",
    "    res = dotp / m_prod\n",
    "\n",
    "    # α = arccos[(a · b) / (|a| * |b|)]\n",
    "    an = np.arccos(res)\n",
    "\n",
    "    # calculated angle from radians to degrees\n",
    "    return np.degrees(an)\n",
    "\n",
    "\n",
    "def parse_cordinates(cords):\n",
    "    \"\"\"\n",
    "    Get N cordinate values given the string representation \"(x, y, z, w...)\".\n",
    "\n",
    "    Parameters:\n",
    "        cords (string): Cordinates expressed as (x, y, z, w...)\n",
    "    Returns:\n",
    "        list: Parsed cordinates, N float values.\n",
    "    \"\"\"\n",
    "    # remove parenthesis from string\n",
    "    cords = cords.replace(\"(\", \"\").replace(\")\", \"\")\n",
    "    # split string by comma, cast values as floats\n",
    "    return [float(v) for v in cords.split(\",\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Read participants cleaned data__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>expo</th>\n",
       "      <th>side</th>\n",
       "      <th>condition</th>\n",
       "      <th>questionnaire</th>\n",
       "      <th>nulls_%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4f583872ffed48249874b00f3d389cfc</th>\n",
       "      <td>2019-05-14 13:21:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>No</td>\n",
       "      <td>1.610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33d68ad55ef544fab9fd08108e3913ca</th>\n",
       "      <td>2019-05-14 16:44:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>No</td>\n",
       "      <td>0.037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8598a83e2af441b8bcd0ae5d84beb875</th>\n",
       "      <td>2019-05-14 16:51:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>RadioTalk</td>\n",
       "      <td>No</td>\n",
       "      <td>0.037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37d7e67934974217830bb429bba7fd76</th>\n",
       "      <td>2019-05-14 16:54:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>left</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>No</td>\n",
       "      <td>0.824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42bac596059749b5b8e8e83ae61de9b4</th>\n",
       "      <td>2019-05-14 16:56:00</td>\n",
       "      <td>MSW</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>No</td>\n",
       "      <td>0.225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cfe9482181f74f80b88cd4b1c048ab94</th>\n",
       "      <td>2019-12-30 15:29:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>No</td>\n",
       "      <td>0.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611d35a7c3e940cc82495e53c2a8532d</th>\n",
       "      <td>2020-01-03 16:12:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>No</td>\n",
       "      <td>0.674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3b6fda285d9e412eb081986b2f22a2e3</th>\n",
       "      <td>2020-01-03 16:13:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>left</td>\n",
       "      <td>AVAS</td>\n",
       "      <td>No</td>\n",
       "      <td>1.797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18ffb0abdc8642098c479380bfa533d1</th>\n",
       "      <td>2020-01-03 16:15:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>left</td>\n",
       "      <td>RadioTalk</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c06f123b35b74bb489ec239b1cac9eb4</th>\n",
       "      <td>2020-01-03 16:16:00</td>\n",
       "      <td>BMBF</td>\n",
       "      <td>right</td>\n",
       "      <td>TaxiDriver</td>\n",
       "      <td>No</td>\n",
       "      <td>0.187</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24356 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 date  expo   side  \\\n",
       "id                                                                   \n",
       "4f583872ffed48249874b00f3d389cfc  2019-05-14 13:21:00   MSW  right   \n",
       "33d68ad55ef544fab9fd08108e3913ca  2019-05-14 16:44:00   MSW  right   \n",
       "8598a83e2af441b8bcd0ae5d84beb875  2019-05-14 16:51:00   MSW  right   \n",
       "37d7e67934974217830bb429bba7fd76  2019-05-14 16:54:00   MSW   left   \n",
       "42bac596059749b5b8e8e83ae61de9b4  2019-05-14 16:56:00   MSW  right   \n",
       "...                                               ...   ...    ...   \n",
       "cfe9482181f74f80b88cd4b1c048ab94  2019-12-30 15:29:00  BMBF  right   \n",
       "611d35a7c3e940cc82495e53c2a8532d  2020-01-03 16:12:00  BMBF  right   \n",
       "3b6fda285d9e412eb081986b2f22a2e3  2020-01-03 16:13:00  BMBF   left   \n",
       "18ffb0abdc8642098c479380bfa533d1  2020-01-03 16:15:00  BMBF   left   \n",
       "c06f123b35b74bb489ec239b1cac9eb4  2020-01-03 16:16:00  BMBF  right   \n",
       "\n",
       "                                   condition questionnaire  nulls_%  \n",
       "id                                                                   \n",
       "4f583872ffed48249874b00f3d389cfc  TaxiDriver            No    1.610  \n",
       "33d68ad55ef544fab9fd08108e3913ca        AVAS            No    0.037  \n",
       "8598a83e2af441b8bcd0ae5d84beb875   RadioTalk            No    0.037  \n",
       "37d7e67934974217830bb429bba7fd76        AVAS            No    0.824  \n",
       "42bac596059749b5b8e8e83ae61de9b4  TaxiDriver            No    0.225  \n",
       "...                                      ...           ...      ...  \n",
       "cfe9482181f74f80b88cd4b1c048ab94        AVAS            No    0.300  \n",
       "611d35a7c3e940cc82495e53c2a8532d  TaxiDriver            No    0.674  \n",
       "3b6fda285d9e412eb081986b2f22a2e3        AVAS            No    1.797  \n",
       "18ffb0abdc8642098c479380bfa533d1   RadioTalk           Yes    0.150  \n",
       "c06f123b35b74bb489ec239b1cac9eb4  TaxiDriver            No    0.187  \n",
       "\n",
       "[24356 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parts = pd.read_csv(\"./participants_clean.csv\")\n",
    "parts = parts.set_index(\"id\")  # set uid column as index (remove default)\n",
    "parts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Participant data example to process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trackerPosition</th>\n",
       "      <th>trackerRotation</th>\n",
       "      <th>noseVector</th>\n",
       "      <th>hitObjectNames</th>\n",
       "      <th>hitObjectPositions</th>\n",
       "      <th>hitPositionOnObjects</th>\n",
       "      <th>hitObjectGroups</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>frameNumber</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>(-193.8, 102.3, -131.8)</td>\n",
       "      <td>(0.0, 0.7, 0.1, 0.7)</td>\n",
       "      <td>(1.0, 0.2, 0.1)</td>\n",
       "      <td>Start, tree_2 (10), Sky_side</td>\n",
       "      <td>(-166.9, 99.11023, -132), (74.1333, 114.3709, ...</td>\n",
       "      <td>(-177.7123, 104.7654, -130.4461), (73.2329, 14...</td>\n",
       "      <td>Start, WestRide Trees/Rocks/Nature, Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>(-193.8, 102.3, -131.8)</td>\n",
       "      <td>(0.0, 0.7, 0.1, 0.7)</td>\n",
       "      <td>(1.0, 0.2, 0.1)</td>\n",
       "      <td>Start, tree_2 (10), Sky_side</td>\n",
       "      <td>(-166.9, 99.11023, -132), (74.1333, 114.3709, ...</td>\n",
       "      <td>(-177.7123, 104.8219, -130.527), (73.2329, 144...</td>\n",
       "      <td>Start, WestRide Trees/Rocks/Nature, Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>(-193.8, 102.3, -131.8)</td>\n",
       "      <td>(0.0, 0.7, 0.1, 0.7)</td>\n",
       "      <td>(1.0, 0.2, 0.1)</td>\n",
       "      <td>Start, tree_2 (10), Sky_side</td>\n",
       "      <td>(-166.9, 99.11023, -132), (74.1333, 114.3709, ...</td>\n",
       "      <td>(-177.7123, 104.7937, -130.5624), (73.77499, 1...</td>\n",
       "      <td>Start, WestRide Trees/Rocks/Nature, Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>(-193.7, 102.3, -131.8)</td>\n",
       "      <td>(0.0, 0.7, 0.1, 0.7)</td>\n",
       "      <td>(1.0, 0.2, 0.1)</td>\n",
       "      <td>Start, Sky_side</td>\n",
       "      <td>(-166.9, 99.11023, -132), (775.8169, 268.5637,...</td>\n",
       "      <td>(-177.7123, 104.7296, -130.5805), (775.8169, 2...</td>\n",
       "      <td>Start, Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>(-193.7, 102.3, -131.8)</td>\n",
       "      <td>(0.0, 0.7, 0.1, 0.7)</td>\n",
       "      <td>(1.0, 0.2, 0.1)</td>\n",
       "      <td>Start, Sky_side</td>\n",
       "      <td>(-166.9, 99.11023, -132), (775.8169, 268.5637,...</td>\n",
       "      <td>(-177.7123, 104.6884, -130.6121), (775.8169, 2...</td>\n",
       "      <td>Start, Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3066</th>\n",
       "      <td>(-41.4, 102.2, -199.3)</td>\n",
       "      <td>(0.0, 0.8, 0.0, -0.7)</td>\n",
       "      <td>(-1.0, 0.0, -0.1)</td>\n",
       "      <td>Sky_side, 3841_combined_LB_building-2  [Legacy...</td>\n",
       "      <td>(-769.1831, 268.5637, -11.36621), (-89.89999, ...</td>\n",
       "      <td>(-769.1831, 101.5359, -305.9648), (-71.96298, ...</td>\n",
       "      <td>Sky, WestRide Buildings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3067</th>\n",
       "      <td>(-41.7, 102.2, -199.3)</td>\n",
       "      <td>(0.0, 0.8, 0.0, -0.7)</td>\n",
       "      <td>(-1.0, 0.0, -0.1)</td>\n",
       "      <td>Sky_side, 3841_combined_LB_building-2  [Legacy...</td>\n",
       "      <td>(-769.1831, 268.5637, -11.36621), (-89.89999, ...</td>\n",
       "      <td>(-769.1831, 101.6931, -300.4803), (-71.96298, ...</td>\n",
       "      <td>Sky, WestRide Buildings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3068</th>\n",
       "      <td>(-41.9, 102.2, -199.3)</td>\n",
       "      <td>(0.0, 0.8, 0.0, -0.7)</td>\n",
       "      <td>(-1.0, 0.0, -0.1)</td>\n",
       "      <td>Sky_side, 3841_combined_LB_building-2  [Legacy...</td>\n",
       "      <td>(-769.1831, 268.5637, -11.36621), (-89.89999, ...</td>\n",
       "      <td>(-769.1831, 101.9324, -297.968), (-71.96298, 1...</td>\n",
       "      <td>Sky, WestRide Buildings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3069</th>\n",
       "      <td>(-42.1, 102.2, -199.3)</td>\n",
       "      <td>(0.0, 0.8, 0.0, -0.7)</td>\n",
       "      <td>(-1.0, 0.0, -0.1)</td>\n",
       "      <td>Sky_side, 3841_combined_LB_building-2  [Legacy...</td>\n",
       "      <td>(-769.1831, 268.5637, -11.36621), (-89.89999, ...</td>\n",
       "      <td>(-769.1831, 101.8583, -293.1292), (-71.96298, ...</td>\n",
       "      <td>Sky, WestRide Buildings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3070</th>\n",
       "      <td>(-42.3, 102.2, -199.3)</td>\n",
       "      <td>(0.0, 0.7, 0.0, -0.7)</td>\n",
       "      <td>(-1.0, 0.0, -0.1)</td>\n",
       "      <td>Sky_side, 3841_combined_LB_building-2  [Legacy...</td>\n",
       "      <td>(-769.1831, 268.5637, -11.36621), (-89.89999, ...</td>\n",
       "      <td>(-769.1831, 101.8903, -290.9534), (-71.96298, ...</td>\n",
       "      <td>Sky, WestRide Buildings</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2671 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     trackerPosition        trackerRotation  \\\n",
       "frameNumber                                                   \n",
       "400          (-193.8, 102.3, -131.8)   (0.0, 0.7, 0.1, 0.7)   \n",
       "401          (-193.8, 102.3, -131.8)   (0.0, 0.7, 0.1, 0.7)   \n",
       "402          (-193.8, 102.3, -131.8)   (0.0, 0.7, 0.1, 0.7)   \n",
       "403          (-193.7, 102.3, -131.8)   (0.0, 0.7, 0.1, 0.7)   \n",
       "404          (-193.7, 102.3, -131.8)   (0.0, 0.7, 0.1, 0.7)   \n",
       "...                              ...                    ...   \n",
       "3066          (-41.4, 102.2, -199.3)  (0.0, 0.8, 0.0, -0.7)   \n",
       "3067          (-41.7, 102.2, -199.3)  (0.0, 0.8, 0.0, -0.7)   \n",
       "3068          (-41.9, 102.2, -199.3)  (0.0, 0.8, 0.0, -0.7)   \n",
       "3069          (-42.1, 102.2, -199.3)  (0.0, 0.8, 0.0, -0.7)   \n",
       "3070          (-42.3, 102.2, -199.3)  (0.0, 0.7, 0.0, -0.7)   \n",
       "\n",
       "                    noseVector  \\\n",
       "frameNumber                      \n",
       "400            (1.0, 0.2, 0.1)   \n",
       "401            (1.0, 0.2, 0.1)   \n",
       "402            (1.0, 0.2, 0.1)   \n",
       "403            (1.0, 0.2, 0.1)   \n",
       "404            (1.0, 0.2, 0.1)   \n",
       "...                        ...   \n",
       "3066         (-1.0, 0.0, -0.1)   \n",
       "3067         (-1.0, 0.0, -0.1)   \n",
       "3068         (-1.0, 0.0, -0.1)   \n",
       "3069         (-1.0, 0.0, -0.1)   \n",
       "3070         (-1.0, 0.0, -0.1)   \n",
       "\n",
       "                                                hitObjectNames  \\\n",
       "frameNumber                                                      \n",
       "400                               Start, tree_2 (10), Sky_side   \n",
       "401                               Start, tree_2 (10), Sky_side   \n",
       "402                               Start, tree_2 (10), Sky_side   \n",
       "403                                            Start, Sky_side   \n",
       "404                                            Start, Sky_side   \n",
       "...                                                        ...   \n",
       "3066         Sky_side, 3841_combined_LB_building-2  [Legacy...   \n",
       "3067         Sky_side, 3841_combined_LB_building-2  [Legacy...   \n",
       "3068         Sky_side, 3841_combined_LB_building-2  [Legacy...   \n",
       "3069         Sky_side, 3841_combined_LB_building-2  [Legacy...   \n",
       "3070         Sky_side, 3841_combined_LB_building-2  [Legacy...   \n",
       "\n",
       "                                            hitObjectPositions  \\\n",
       "frameNumber                                                      \n",
       "400          (-166.9, 99.11023, -132), (74.1333, 114.3709, ...   \n",
       "401          (-166.9, 99.11023, -132), (74.1333, 114.3709, ...   \n",
       "402          (-166.9, 99.11023, -132), (74.1333, 114.3709, ...   \n",
       "403          (-166.9, 99.11023, -132), (775.8169, 268.5637,...   \n",
       "404          (-166.9, 99.11023, -132), (775.8169, 268.5637,...   \n",
       "...                                                        ...   \n",
       "3066         (-769.1831, 268.5637, -11.36621), (-89.89999, ...   \n",
       "3067         (-769.1831, 268.5637, -11.36621), (-89.89999, ...   \n",
       "3068         (-769.1831, 268.5637, -11.36621), (-89.89999, ...   \n",
       "3069         (-769.1831, 268.5637, -11.36621), (-89.89999, ...   \n",
       "3070         (-769.1831, 268.5637, -11.36621), (-89.89999, ...   \n",
       "\n",
       "                                          hitPositionOnObjects  \\\n",
       "frameNumber                                                      \n",
       "400          (-177.7123, 104.7654, -130.4461), (73.2329, 14...   \n",
       "401          (-177.7123, 104.8219, -130.527), (73.2329, 144...   \n",
       "402          (-177.7123, 104.7937, -130.5624), (73.77499, 1...   \n",
       "403          (-177.7123, 104.7296, -130.5805), (775.8169, 2...   \n",
       "404          (-177.7123, 104.6884, -130.6121), (775.8169, 2...   \n",
       "...                                                        ...   \n",
       "3066         (-769.1831, 101.5359, -305.9648), (-71.96298, ...   \n",
       "3067         (-769.1831, 101.6931, -300.4803), (-71.96298, ...   \n",
       "3068         (-769.1831, 101.9324, -297.968), (-71.96298, 1...   \n",
       "3069         (-769.1831, 101.8583, -293.1292), (-71.96298, ...   \n",
       "3070         (-769.1831, 101.8903, -290.9534), (-71.96298, ...   \n",
       "\n",
       "                                     hitObjectGroups  \n",
       "frameNumber                                           \n",
       "400          Start, WestRide Trees/Rocks/Nature, Sky  \n",
       "401          Start, WestRide Trees/Rocks/Nature, Sky  \n",
       "402          Start, WestRide Trees/Rocks/Nature, Sky  \n",
       "403                                       Start, Sky  \n",
       "404                                       Start, Sky  \n",
       "...                                              ...  \n",
       "3066                         Sky, WestRide Buildings  \n",
       "3067                         Sky, WestRide Buildings  \n",
       "3068                         Sky, WestRide Buildings  \n",
       "3069                         Sky, WestRide Buildings  \n",
       "3070                         Sky, WestRide Buildings  \n",
       "\n",
       "[2671 rows x 7 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected = parts.iloc[0]\n",
    "uid = pd.Index(selected).name\n",
    "\n",
    "file = f\"Path_ MS Wissenschaft-{selected.condition}-{uid}.csv\"\n",
    "df = pd.read_csv(f\"{PATH_RAYC}/{file}\")\n",
    "df = df.set_index(\"frameNumber\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4258e2e6970476da2ab1710ccb9b17f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def process_hits(uid):\n",
    "    \"\"\"\n",
    "    Process raw raycasted hit data as hit origin and information\n",
    "    separately given a participant UID:\n",
    "\n",
    "    - Disentangle origin and direction coordinates\n",
    "    - Disentangle object and group names\n",
    "    - Calculate direction (nose vector) angles and hit distances\n",
    "    - Order object hits within the same frame by distance\n",
    "    - Rename/rearrange object groups and names\n",
    "    - Fix objects belonging to wrong group\n",
    "    - Store as processed hit origin and hit info\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant UID to process.\n",
    "    \"\"\"\n",
    "\n",
    "    cond = parts.loc[uid].condition\n",
    "    file = f\"Path_ MS Wissenschaft-{cond}-{uid}.csv\"\n",
    "    df_part = pd.read_csv(f\"{PATH_RAYC}/{file}\", keep_default_na=False)\n",
    "\n",
    "    # generate the new dict (=> dataFrame) structure\n",
    "    orgs = {c: [] for c in HIT_ORIGIN}\n",
    "    hits = {c: [] for c in HIT_INFO}\n",
    "\n",
    "    # df indices (all frames)\n",
    "    frames = df_part.index.tolist()\n",
    "\n",
    "    for f in frames:  # iterate over frames\n",
    "\n",
    "        cur = df_part.loc[f]  # current row (frame)\n",
    "        # parse and store origin\n",
    "        origin = cur.trackerPosition\n",
    "        origX, origY, origZ = parse_cordinates(origin)\n",
    "        # parse and store rotation\n",
    "        rotation = cur.trackerRotation\n",
    "        rotX, rotY, rotZ, rotW = parse_cordinates(rotation)\n",
    "        # parse and store nose vector (direction)\n",
    "        direction = cur.noseVector\n",
    "        directX, directY, directZ = parse_cordinates(direction)\n",
    "        noseVector = np.array([directX, directY, directZ])\n",
    "        # calculate angles between nose vector and X-Y-Z axes\n",
    "        x_angle = angle(noseVector, X_DIR)\n",
    "        y_angle = angle(noseVector, Y_DIR)\n",
    "        z_angle = angle(noseVector, Z_DIR)\n",
    "\n",
    "        # store hit origin info\n",
    "        orgs[\"frame\"].append(f)\n",
    "        orgs[\"originX\"].append(origX)\n",
    "        orgs[\"originY\"].append(origY)\n",
    "        orgs[\"originZ\"].append(origZ)\n",
    "        orgs[\"rotationX\"].append(rotX)\n",
    "        orgs[\"rotationY\"].append(rotY)\n",
    "        orgs[\"rotationZ\"].append(rotZ)\n",
    "        orgs[\"rotationW\"].append(rotW)\n",
    "        orgs[\"directionX\"].append(directX)\n",
    "        orgs[\"directionY\"].append(directY)\n",
    "        orgs[\"directionZ\"].append(directZ)\n",
    "        # store horizontal and vertical angles\n",
    "        orgs[\"angleX\"].append(x_angle)\n",
    "        orgs[\"angleY\"].append(y_angle)\n",
    "        orgs[\"angleZ\"].append(z_angle)\n",
    "\n",
    "        # get hit names to check hits in frame\n",
    "        names = cur.hitObjectNames\n",
    "        groups = cur.hitObjectGroups\n",
    "        hit_pos = cur.hitPositionOnObjects\n",
    "        obj_pos = cur.hitObjectPositions\n",
    "        if names == \"null\":  # no hit\n",
    "            names = [None]\n",
    "            groups = [None]\n",
    "            dist = None\n",
    "            objX, objY, objZ = [None, None, None]\n",
    "            hitX, hitY, hitZ = [None, None, None]\n",
    "        elif \",\" not in names:  # single hit\n",
    "            # encapsulate single values on lists\n",
    "            names = [names]\n",
    "            groups = [groups]\n",
    "            obj_pos = [obj_pos]\n",
    "            hit_pos = [hit_pos]\n",
    "        else:  # more than 1 hit on the same frame\n",
    "            # parse object names (also remove left and right whitespaces)\n",
    "            names = [n.strip() for n in names.split(\", \")]\n",
    "            # parse object groups (also remove left and right whitespaces)\n",
    "            groups = [g.strip() for g in groups.split(\", \")]\n",
    "            # parse object and hit positions (3D cordinate groups)\n",
    "            obj_pos = obj_pos.split(\"),\")\n",
    "            hit_pos = hit_pos.split(\"),\")\n",
    "\n",
    "        n_objects = len(names)\n",
    "        for i in range(n_objects):  # for each object\n",
    "            # get name and group\n",
    "            name = names[i]\n",
    "            group = groups[i]\n",
    "            if name:\n",
    "                # parse and store object and hit position cordinates\n",
    "                objX, objY, objZ = parse_cordinates(obj_pos[i])\n",
    "                hitX, hitY, hitZ = parse_cordinates(hit_pos[i])\n",
    "                # calculate hit distance\n",
    "                orig = np.array([origX, origY, origZ])\n",
    "                hit = np.array([hitX, hitY, hitZ])\n",
    "                dist = np.linalg.norm(hit - orig)\n",
    "            # store frame number, origin, and direction values\n",
    "            hits[\"frame\"].append(f)\n",
    "            # store object name, group, and distance\n",
    "            hits[\"name\"].append(name)\n",
    "            hits[\"group\"].append(group)\n",
    "            hits[\"distance\"].append(dist)\n",
    "            # store object and hit cordinates\n",
    "            hits[\"objectX\"].append(hitX)\n",
    "            hits[\"objectY\"].append(hitY)\n",
    "            hits[\"objectZ\"].append(hitZ)\n",
    "            hits[\"hitX\"].append(hitX)\n",
    "            hits[\"hitY\"].append(hitY)\n",
    "            hits[\"hitZ\"].append(hitZ)\n",
    "\n",
    "    # dict to df\n",
    "    hits = pd.DataFrame(hits)\n",
    "    orgs = pd.DataFrame(orgs)\n",
    "    orgs = orgs.set_index(\"frame\")\n",
    "\n",
    "    # rename/rearrange group names\n",
    "    hits.group = hits.group.replace(GRP_REPLACE)\n",
    "\n",
    "    # Specific objects in wrong group fixes (car objects inside events and\n",
    "    # street objects, and jogging and walking dudes inside events)\n",
    "    # select indices (True/False) with dynamic car on \"Events\"\n",
    "    car_event = (hits.group == \"Events\") & (hits.name == \"U_SUV03(Clone)\")\n",
    "    # if any dynamic car on \"Events\" set group to \"Dynamic Cars\"\n",
    "    if any(car_event):\n",
    "        # for some strange reason the values of the column name \"group\" cannot\n",
    "        # be modified using a selection of rows. But by modifying it on a copy\n",
    "        # and then reassigning it to the original selection the change works\n",
    "        # also rename \"U_SUV03(Clone)\" to \"fast_car\" for dissambiguiation\n",
    "        # within the group \"Dynamic Cars\"\n",
    "        sel = hits[car_event]\n",
    "        sel.name = \"fast_car\"\n",
    "        sel.group = \"Dynamic Cars\"\n",
    "        hits[car_event] = sel\n",
    "    # apply also name and group a fixes for the jogging and walking dudes\n",
    "    jog_ev = (hits.group == \"Events\") & (\n",
    "        hits.name == \"Event 2.1 (2) Running(Clone)\"\n",
    "    )\n",
    "    if any(jog_ev):\n",
    "        sel = hits[jog_ev]\n",
    "        sel.name = \"jogging_man\"\n",
    "        sel.group = \"Dynamic Pedestrians\"\n",
    "        hits[jog_ev] = sel\n",
    "    wal_ev = (hits.group == \"Events\") & (\n",
    "        hits.name == \"event 2.2 walking(Clone)\"\n",
    "    )\n",
    "    if any(wal_ev):\n",
    "        sel = hits[wal_ev]\n",
    "        sel.name = \"walking_man\"\n",
    "        sel.group = \"Dynamic Pedestrians\"\n",
    "        hits[wal_ev] = sel\n",
    "    # select indices (True/False) with static car on \"Street Objects\"\n",
    "    # object group selection\n",
    "    str_obj = hits.group == \"Street Objects\"\n",
    "    # object name selection\n",
    "    car_in = hits.name.str.contains(\"car (\", regex=False, na=False)\n",
    "    car_str_obj = str_obj & car_in  # combined selection (both conditions)\n",
    "    # if any static car on \"Street Objects\" set group to \"Static Cars\"\n",
    "    if any(car_str_obj):\n",
    "        # same trick as above\n",
    "        sel = hits[car_str_obj]\n",
    "        sel.group = \"Static Cars\"\n",
    "        hits[car_str_obj] = sel\n",
    "\n",
    "    # ensure all hit objects are ordered by distance whitin a frame.\n",
    "    new_idxs = []  # to collect new ordered indices\n",
    "    for f in frames:  # iterate over frames\n",
    "        cur = hits[hits.frame == f]  # select rows with current frame\n",
    "        idxs = cur.index.tolist()  # unordered indices\n",
    "        cur = cur.sort_values(by=\"distance\")  # sort series\n",
    "        ordered = cur.index.tolist()  # ordered indices\n",
    "        for idx in ordered:  # iterate over ordered indices\n",
    "            new_idxs.append(idx)  # store them\n",
    "\n",
    "    # reorder dataframe according to the indices stored\n",
    "    hits = hits.reindex(new_idxs)\n",
    "\n",
    "    # store into CSV\n",
    "    orgs.to_csv(f\"./data/processed/origins/{uid}.csv\")\n",
    "    hits.to_csv(f\"./data/processed/hits/{uid}.csv\", index=False)\n",
    "\n",
    "\n",
    "# initialize a multiprocessing pool with the available cpu cores\n",
    "pool = Pool(processes=CORES)\n",
    "\n",
    "# participants uids to process\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "# participants progress bar\n",
    "parts_progress = tqdm(\n",
    "    iterable=pool.imap_unordered(func=process_hits, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_progress:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Collect and store all object unique groups and names across all participants__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42c86a1ace554c08b7270f17330e3a08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 uids processed:                                                                                …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1542db278156409f849f15aef7dc0e2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd35f26f750642f0b01f60c7c42bce8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 16 📂 groups processed:                                                                                 …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    }
   ],
   "source": [
    "def collect_names(uid):\n",
    "    \"\"\"\n",
    "    Collect all unique object names given a participant UID.\n",
    "    Parameters:\n",
    "        uid (str): Participant UID to process.\n",
    "    \"\"\"\n",
    "    cur = pd.read_csv(f\"{PATH_PROC}/hits/{uid}.csv\")\n",
    "    for grp in groups:\n",
    "        names[uid][grp] = cur[cur.group == grp].name.unique()\n",
    "\n",
    "\n",
    "def unique_names(grp):\n",
    "    \"\"\"\n",
    "    Check all participants and get all unique object names given a group.\n",
    "    Parameters:\n",
    "        grp (str): Object group name.\n",
    "    \"\"\"\n",
    "    for uid in uids:\n",
    "        cum = list(unique[grp])\n",
    "        cur = list(names[uid][grp])\n",
    "        unique[grp] = list(set(cum + cur))\n",
    "\n",
    "\n",
    "groups = [\n",
    "    \"Buildings\",\n",
    "    \"Crosswalks\",\n",
    "    \"Dynamic Cars\",\n",
    "    \"Dynamic Pedestrians\",\n",
    "    \"End\",\n",
    "    \"Events\",\n",
    "    \"Inside Car\",\n",
    "    \"Nature\",\n",
    "    \"Roads\",\n",
    "    \"Sky\",\n",
    "    \"Start\",\n",
    "    \"Static Cars\",\n",
    "    \"Static Pedestrians\",\n",
    "    \"Street Objects\",\n",
    "    \"Traffic Lights\",\n",
    "    \"Traffic Signs\",\n",
    "]\n",
    "\n",
    "manager = Manager()  # manage shared memory types\n",
    "\n",
    "# participants uids to process\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "# uids progress bar\n",
    "uids_pbar = tqdm(\n",
    "    iterable=uids,\n",
    "    total=len(uids),\n",
    "    desc=\"📂 uids\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "gs = {grp: manager.list() for grp in groups}\n",
    "\n",
    "# create memory-shared dict of lists for multiprocessing\n",
    "names = manager.dict({uid: manager.dict(gs) for uid in uids_pbar})\n",
    "unique = manager.dict({grp: manager.list() for grp in groups})\n",
    "\n",
    "l = Lock()\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack, initargs=(l,))\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap(func=collect_names, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()\n",
    "\n",
    "# After collecting them, get all unique names per group across participants\n",
    "# TODO:\n",
    "# - Find a way to see the progress while unifying the groups across parts\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack, initargs=(l,))\n",
    "\n",
    "# groups progress bar\n",
    "groups_pbar = tqdm(\n",
    "    iterable=pool.imap(func=unique_names, iterable=groups),\n",
    "    total=len(groups),\n",
    "    desc=\"📂 groups\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in groups_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cast proxy object/s (otherwise they can't be accessed)\n",
    "names = dict(unique)\n",
    "\n",
    "with open(f\"./groups_and_names.json\", \"w\") as f:\n",
    "    json.dump(names, f, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Compute gaze vs rest of hit events__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5e52d296ccc4cea8a993958fa2fdce7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 0 of 24356 📂 participants processed:                                                                        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     "
     ]
    }
   ],
   "source": [
    "def nearest_hit(df):\n",
    "    \"\"\"\n",
    "    Add description.\n",
    "\n",
    "    Parameters:\n",
    "        df (pd.DataFrame): Add desc.\n",
    "    Returns:\n",
    "        found (bool): Add desc.\n",
    "        row (pd.Series): Add desc.\n",
    "    \"\"\"\n",
    "    idxs = df.index.tolist()\n",
    "    found = False\n",
    "    for idx in idxs:\n",
    "        c = df.loc[idx]\n",
    "        row = c if c.group not in EXCEPT_GROUPS and c.group != \"\" else None\n",
    "        found = not isinstance(row, type(None))\n",
    "        if found:\n",
    "            break\n",
    "    return found, row\n",
    "\n",
    "\n",
    "def collect_hit_events(uid):\n",
    "    \"\"\"\n",
    "    Collect hit (saccade/gaze) events and related info of a participant given\n",
    "    its UID.\n",
    "\n",
    "    Parameters:\n",
    "        uid (str): Participant unique identifier.\n",
    "    \"\"\"\n",
    "    # read current participant origins and hits datasets\n",
    "    orgs = pd.read_csv(f\"{PATH_PROC}/origins/{uid}.csv\")\n",
    "    orgs = orgs.set_index(\"frame\")  # set frame column as index\n",
    "    hits = pd.read_csv(f\"{PATH_PROC}/hits/{uid}.csv\")\n",
    "\n",
    "    frames = orgs.index.tolist()  # all frames (0-2670)\n",
    "\n",
    "    # dict to collect gaze events info\n",
    "    h_events = {\n",
    "        \"name\": [],\n",
    "        \"group\": [],\n",
    "        \"start\": [],\n",
    "        \"end\": [],\n",
    "        \"nHits\": [],\n",
    "        \"distance\": [],\n",
    "        \"length\": [],\n",
    "        \"type\": [],\n",
    "    }\n",
    "\n",
    "    # dict to check consecutives\n",
    "    check = {\"p\": None, \"start\": None, \"cnt\": 0}\n",
    "    dist = []\n",
    "    for f in frames:  # for each frame\n",
    "        cur = hits[hits.frame == f]\n",
    "        # closest object hit if not in exceptions nor null, otherwise next\n",
    "        found, cur = nearest_hit(cur)\n",
    "        if found:\n",
    "            d = float(cur.distance)\n",
    "            name = cur[\"name\"]  # .name calls .Name of the series!!\n",
    "            group = cur.group\n",
    "            if f < len(frames) - 1:\n",
    "                fou, nex = nearest_hit(hits[hits.frame == f + 1])\n",
    "                nex_name = nex[\"name\"] if fou else None\n",
    "                nex_grp = nex.group if fou else None\n",
    "            else:\n",
    "                nex_name = None\n",
    "            if not check[\"p\"]:  # not previous yet or ATM\n",
    "                dist = []\n",
    "                dist.append(d)\n",
    "                check[\"p\"] = name\n",
    "                check[\"start\"] = f\n",
    "                check[\"cnt\"] += 1\n",
    "            else:\n",
    "                dist.append(d)\n",
    "                check[\"p\"] = name\n",
    "                check[\"cnt\"] += 1\n",
    "            if name != nex_name:\n",
    "                d = sum(dist) / len(dist)\n",
    "                d = round(d, 3)\n",
    "                n_hits = check[\"cnt\"]\n",
    "                e_type = \"gaze\" if n_hits > 7 else \"rest\"\n",
    "                h_events[\"name\"].append(name)\n",
    "                h_events[\"group\"].append(group)\n",
    "                h_events[\"start\"].append(check[\"start\"])\n",
    "                h_events[\"end\"].append(f)\n",
    "                h_events[\"nHits\"].append(n_hits)\n",
    "                h_events[\"distance\"].append(d)\n",
    "                h_events[\"length\"].append(n_hits * 0.033)\n",
    "                h_events[\"type\"].append(e_type)\n",
    "                # reset\n",
    "                check = {\"p\": None, \"start\": None, \"cnt\": 0}\n",
    "                store = False\n",
    "\n",
    "    h_events = pd.DataFrame(h_events)\n",
    "    h_events.to_csv(f\"{PATH_PROC}/hit_events/{uid}.csv\", index=False)\n",
    "\n",
    "\n",
    "# initialize pool of processes according to the available cpu core threads\n",
    "pool = Pool(processes=CORES, initializer=pbar_fork_hack)\n",
    "\n",
    "# participants ids\n",
    "uids = parts.index.tolist()\n",
    "\n",
    "# participants progress bar\n",
    "parts_pbar = tqdm(\n",
    "    iterable=pool.imap_unordered(func=collect_hit_events, iterable=uids),\n",
    "    total=len(uids),\n",
    "    desc=\"📂 participants\",\n",
    "    dynamic_ncols=True,\n",
    "    bar_format=B_FORMAT,\n",
    ")\n",
    "\n",
    "# loop necessary for displaying properly the progressbar with multiprocessing\n",
    "# source: https://stackoverflow.com/a/40133278\n",
    "for _ in parts_pbar:\n",
    "    pass\n",
    "\n",
    "# close pool instance, no more work to submit\n",
    "pool.close()\n",
    "# wait for the worker processes to terminate\n",
    "pool.join()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (msw-bmbf_da)",
   "language": "python",
   "name": "msw-bmbf_da"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}