extended_generate_supersegment_speed_feats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3bfe7713-e3f5-4949-abf4-b2078b0adc91",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import re\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from typing import Callable\n",
    "\n",
    "from utils import load_edges, proba_to_logit, get_weights_from_class_fractions, load_labels_core, load_supersegments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1d717fd9-994d-4992-9e12-ac93302d502f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Config\n",
    "# data_dir = Path(\"/Users/martin/PycharmProjects/traffic4cast/data/\")\n",
    "data_dir = Path(\"/Users/andrei/Desktop/data4cast/data/\")\n",
    "\n",
    "city_name = \"melbourne\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42414256-c635-4e9e-8262-6517b84e3649",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c1ce7885-67dd-468a-a860-182d7a9a39e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2d0726f3b4b44c48d78ad2a388fa4e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4012 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3c8f1153064a04a3511b5e028ebcd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4012 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nodes = pd.read_parquet(data_dir / f\"road_graph/{city_name}/road_graph_nodes.parquet\")\n",
    "node_coordinates = nodes.groupby(\"node_id\")[[\"x\", \"y\"]].first().to_dict(orient=\"index\")\n",
    "node_to_lat_lng = nodes.set_index(\"node_id\")[[\"x\", \"y\"]].T.to_dict()\n",
    "\n",
    "supersegments, supersegment_to_id, id_to_supersegment = load_supersegments(city_name, node_coordinates, del_segment_feats=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ffc8208-af77-4238-81b9-6ebc288c215b",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges, edge_id_to_int, edge_int_to_id = load_edges(city_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "37adf498-9bba-451f-a2aa-310ff7bf1cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "speeds = pd.read_parquet(f\"/Users/andrei/Desktop/data4cast/data/speed_classes/{city_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "227614af-ae65-4094-befe-71f08be3fe6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_to_edges_mapping = [[(u, v) for u, v in zip(r[\"nodes\"], r[\"nodes\"][1:])] for r in supersegments.to_dict(\"records\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fb310bf2-a9bc-4e25-9db2-4cf21b6b75a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments.drop(columns=[\"identifier\", \"nodes\", \"coord_list\", \"representative_point\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d4df4319-eca4-4875-9b8a-991be0bebace",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments[\"edges\"] = supersegment_to_edges_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7d0d899d-55c8-4f13-bd18-45aa0a174e2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>supersegment_id</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>edges</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.292401</td>\n",
       "      <td>51.528028</td>\n",
       "      <td>[(9596340681, 9241203), (9241203, 9596340679),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.160380</td>\n",
       "      <td>51.501745</td>\n",
       "      <td>[(1504500003, 110060), (110060, 236894314), (2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.035030</td>\n",
       "      <td>51.474974</td>\n",
       "      <td>[(6245856720, 6245836252), (6245836252, 171857...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-0.124077</td>\n",
       "      <td>51.501483</td>\n",
       "      <td>[(4765087519292221207, 3237371488848642588), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>-0.128442</td>\n",
       "      <td>51.614820</td>\n",
       "      <td>[(1745361197, 1745361160), (1745361160, 196400...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4007</th>\n",
       "      <td>4007</td>\n",
       "      <td>-0.017609</td>\n",
       "      <td>51.444752</td>\n",
       "      <td>[(620089, 620075), (620075, 432825), (432825, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4008</th>\n",
       "      <td>4008</td>\n",
       "      <td>-0.204618</td>\n",
       "      <td>51.588884</td>\n",
       "      <td>[(196143, 2453134652), (2453134652, 1692087081...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4009</th>\n",
       "      <td>4009</td>\n",
       "      <td>-0.123329</td>\n",
       "      <td>51.509656</td>\n",
       "      <td>[(32925453, 107738), (107738, 107733), (107733...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4010</th>\n",
       "      <td>4010</td>\n",
       "      <td>-0.162744</td>\n",
       "      <td>51.467241</td>\n",
       "      <td>[(1155388513, 1155388654), (1155388654, 264866...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011</th>\n",
       "      <td>4011</td>\n",
       "      <td>0.000751</td>\n",
       "      <td>51.513256</td>\n",
       "      <td>[(641481688, 361257219), (361257219, 11060850)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4012 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      supersegment_id         x          y  \\\n",
       "0                   0 -0.292401  51.528028   \n",
       "1                   1 -0.160380  51.501745   \n",
       "2                   2 -0.035030  51.474974   \n",
       "3                   3 -0.124077  51.501483   \n",
       "4                   4 -0.128442  51.614820   \n",
       "...               ...       ...        ...   \n",
       "4007             4007 -0.017609  51.444752   \n",
       "4008             4008 -0.204618  51.588884   \n",
       "4009             4009 -0.123329  51.509656   \n",
       "4010             4010 -0.162744  51.467241   \n",
       "4011             4011  0.000751  51.513256   \n",
       "\n",
       "                                                  edges  \n",
       "0     [(9596340681, 9241203), (9241203, 9596340679),...  \n",
       "1     [(1504500003, 110060), (110060, 236894314), (2...  \n",
       "2     [(6245856720, 6245836252), (6245836252, 171857...  \n",
       "3     [(4765087519292221207, 3237371488848642588), (...  \n",
       "4     [(1745361197, 1745361160), (1745361160, 196400...  \n",
       "...                                                 ...  \n",
       "4007  [(620089, 620075), (620075, 432825), (432825, ...  \n",
       "4008  [(196143, 2453134652), (2453134652, 1692087081...  \n",
       "4009  [(32925453, 107738), (107738, 107733), (107733...  \n",
       "4010  [(1155388513, 1155388654), (1155388654, 264866...  \n",
       "4011  [(641481688, 361257219), (361257219, 11060850)...  \n",
       "\n",
       "[4012 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "supersegments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5d313cae-a9cd-408f-889e-131842c9e2dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>parsed_maxspeed</th>\n",
       "      <th>speed_kph</th>\n",
       "      <th>importance</th>\n",
       "      <th>highway</th>\n",
       "      <th>oneway</th>\n",
       "      <th>lanes</th>\n",
       "      <th>tunnel</th>\n",
       "      <th>length_meters</th>\n",
       "      <th>counter_distance</th>\n",
       "      <th>edge_id</th>\n",
       "      <th>edge_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>19.402386</td>\n",
       "      <td>6</td>\n",
       "      <td>78112_25508583</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508584</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>63.881347</td>\n",
       "      <td>4</td>\n",
       "      <td>78112_25508584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>78112</td>\n",
       "      <td>3257621681005534125</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0</td>\n",
       "      <td>residential</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>82.385612</td>\n",
       "      <td>5</td>\n",
       "      <td>78112_3257621681005534125</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>99936</td>\n",
       "      <td>2146383887</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>8.076410</td>\n",
       "      <td>1</td>\n",
       "      <td>99936_2146383887</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>99936</td>\n",
       "      <td>4544836433</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>0</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>46.803240</td>\n",
       "      <td>0</td>\n",
       "      <td>99936_4544836433</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132409</th>\n",
       "      <td>4595139612105786518</td>\n",
       "      <td>8842311879</td>\n",
       "      <td>35.1</td>\n",
       "      <td>35.1</td>\n",
       "      <td>0</td>\n",
       "      <td>residential</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>13.486853</td>\n",
       "      <td>4</td>\n",
       "      <td>4595139612105786518_8842311879</td>\n",
       "      <td>132409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132410</th>\n",
       "      <td>8230831116681660864</td>\n",
       "      <td>1149426165</td>\n",
       "      <td>35.1</td>\n",
       "      <td>35.1</td>\n",
       "      <td>0</td>\n",
       "      <td>residential</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>60.246550</td>\n",
       "      <td>9</td>\n",
       "      <td>8230831116681660864_1149426165</td>\n",
       "      <td>132410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132411</th>\n",
       "      <td>1688447984145568529</td>\n",
       "      <td>26559620</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>4.845324</td>\n",
       "      <td>1</td>\n",
       "      <td>1688447984145568529_26559620</td>\n",
       "      <td>132411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132412</th>\n",
       "      <td>3771856370570656347</td>\n",
       "      <td>9402106041</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>8.118341</td>\n",
       "      <td>2</td>\n",
       "      <td>3771856370570656347_9402106041</td>\n",
       "      <td>132412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132413</th>\n",
       "      <td>4890701424133264627</td>\n",
       "      <td>27596189</td>\n",
       "      <td>32.2</td>\n",
       "      <td>32.2</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>7.387061</td>\n",
       "      <td>4</td>\n",
       "      <td>4890701424133264627_27596189</td>\n",
       "      <td>132413</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>132414 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          u                    v  parsed_maxspeed  speed_kph  \\\n",
       "0                     78112             25508583             32.2       32.2   \n",
       "1                     78112             25508584             32.2       32.2   \n",
       "2                     78112  3257621681005534125             32.2       32.2   \n",
       "3                     99936           2146383887             32.2       32.2   \n",
       "4                     99936           4544836433             32.2       32.2   \n",
       "...                     ...                  ...              ...        ...   \n",
       "132409  4595139612105786518           8842311879             35.1       35.1   \n",
       "132410  8230831116681660864           1149426165             35.1       35.1   \n",
       "132411  1688447984145568529             26559620             32.2       32.2   \n",
       "132412  3771856370570656347           9402106041             20.0       20.0   \n",
       "132413  4890701424133264627             27596189             32.2       32.2   \n",
       "\n",
       "        importance       highway  oneway lanes tunnel  length_meters  \\\n",
       "0                0  unclassified   False                   19.402386   \n",
       "1                0  unclassified   False                   63.881347   \n",
       "2                0   residential    True                   82.385612   \n",
       "3                0  unclassified   False                    8.076410   \n",
       "4                0  unclassified    True                   46.803240   \n",
       "...            ...           ...     ...   ...    ...            ...   \n",
       "132409           0   residential   False                   13.486853   \n",
       "132410           0   residential   False                   60.246550   \n",
       "132411           3       primary    True     2              4.845324   \n",
       "132412           0  unclassified    True                    8.118341   \n",
       "132413           3       primary    True     1              7.387061   \n",
       "\n",
       "        counter_distance                         edge_id  edge_int  \n",
       "0                      6                  78112_25508583         0  \n",
       "1                      4                  78112_25508584         1  \n",
       "2                      5       78112_3257621681005534125         2  \n",
       "3                      1                99936_2146383887         3  \n",
       "4                      0                99936_4544836433         4  \n",
       "...                  ...                             ...       ...  \n",
       "132409                 4  4595139612105786518_8842311879    132409  \n",
       "132410                 9  8230831116681660864_1149426165    132410  \n",
       "132411                 1    1688447984145568529_26559620    132411  \n",
       "132412                 2  3771856370570656347_9402106041    132412  \n",
       "132413                 4    4890701424133264627_27596189    132413  \n",
       "\n",
       "[132414 rows x 13 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "90b85175-9fc1-46dc-b988-96649170deb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges = supersegments.explode(\"edges\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2d281498-303b-455e-960a-454e0d42b64d",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges[\"edge_id\"] = [f\"{x[0]}_{x[1]}\" for x in supersegments_edges[\"edges\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b960f1fb-90d3-4a47-b806-ed375f3275b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments = pd.read_parquet(data_dir / f\"road_graph/{city_name}/road_graph_supersegments.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9ba9a916-3b41-422b-82f5-1ad8bf97c513",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9596340681,    9241203, 9596340679,    9241200,   32553207,\n",
       "           197271,     197240,   36679106,   36679116,     197222,\n",
       "       1016971521,     197215,     197210,     197209,     197207,\n",
       "           197206,     197298,     197211, 3146836855,     197290,\n",
       "         71046904,     197292,   71046378,  324012957,     234219,\n",
       "           234222,     234223,  774483306,     234224,   71051657,\n",
       "           234225,     234227,     234237,     224790,  254682567])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "supersegments.iloc[0][\"nodes\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "efe7aede-dc03-4ae6-bed6-c5e2ddbb6ac9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>day</th>\n",
       "      <th>t</th>\n",
       "      <th>volume_class</th>\n",
       "      <th>median_speed_kph</th>\n",
       "      <th>free_flow_kph</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>19.764706</td>\n",
       "      <td>36.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>28</td>\n",
       "      <td>5</td>\n",
       "      <td>27.882353</td>\n",
       "      <td>36.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>46.823529</td>\n",
       "      <td>36.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>36.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>78112</td>\n",
       "      <td>25508583</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>34</td>\n",
       "      <td>5</td>\n",
       "      <td>38.470588</td>\n",
       "      <td>36.352941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       u         v         day   t  volume_class  median_speed_kph  \\\n",
       "0  78112  25508583  2019-07-01   9             3         19.764706   \n",
       "1  78112  25508583  2019-07-01  28             5         27.882353   \n",
       "2  78112  25508583  2019-07-01  29             5         46.823529   \n",
       "3  78112  25508583  2019-07-01  30             3         24.000000   \n",
       "4  78112  25508583  2019-07-01  34             5         38.470588   \n",
       "\n",
       "   free_flow_kph  \n",
       "0      36.352941  \n",
       "1      36.352941  \n",
       "2      36.352941  \n",
       "3      36.352941  \n",
       "4      36.352941  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "speeds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "aecf441f-0876-4a46-b11c-482262acf7a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "337145078it [04:10, 1345293.65it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 337145078/337145078 [02:31<00:00, 2231743.15it/s]\n"
     ]
    }
   ],
   "source": [
    "speeds[\"edge_id\"] = [f\"{u}_{v}\" for u, v in tqdm(zip(speeds[\"u\"], speeds[\"v\"]))]\n",
    "speeds[\"edge_int\"] = [edge_id_to_int[eid] for eid in tqdm(speeds[\"edge_id\"])]\n",
    "# del speeds[\"edge_id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ef32165c-4e33-470d-ae3e-c737a53e3638",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_speeds = speeds.groupby([\"edge_int\"])[[\"volume_class\", \"median_speed_kph\", \"free_flow_kph\"]].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "db98fa51-a6e1-44a4-a9c3-a18e90484bc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges_with_speeds = edges.drop(columns=[\"u\", \"v\", \"edge_id\"]).set_index(\"edge_int\").join(mean_speeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5fb8af36-9258-4c59-8603-c6788e8b67ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges_with_speeds[[\"volume_class\", \"median_speed_kph\", \"free_flow_kph\"]] = \\\n",
    "    edges_with_speeds[[\"volume_class\", \"median_speed_kph\", \"free_flow_kph\"]].fillna(\n",
    "    edges_with_speeds[[\"volume_class\", \"median_speed_kph\", \"free_flow_kph\"]].median()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "155488c2-6c7f-4de4-af65-c547ea2e2c8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges[\"edge_int\"] = [edge_id_to_int[x] for x in supersegments_edges[\"edge_id\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0dfb311b-dd10-4470-8dd8-3946ecdfbfe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges_speeds = supersegments_edges.merge(edges_with_speeds, left_on=\"edge_int\", right_index=True).drop(columns=[\"edges\", \"edge_id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b540456f-07d9-47ee-9f43-d98dc5f1915f",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges_speeds[\"dummy_eta\"] = supersegments_edges_speeds[\"length_meters\"] / (supersegments_edges_speeds[\"median_speed_kph\"] * 1000) * 3600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f1de5306-9a58-45d7-b8a6-e95927c934fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_edges_speeds[\"dummy_eta_freeflow\"] = supersegments_edges_speeds[\"length_meters\"] / (supersegments_edges_speeds[\"free_flow_kph\"] * 1000) * 3600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dd3d69c6-83a8-469c-887f-2e2784251065",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_dummy_eta = supersegments_edges_speeds.groupby(\"supersegment_id\")[\"dummy_eta\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "48f49f0c-cd78-4cef-aeb7-680a9440e689",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_dummy_eta_freeflow = supersegments_edges_speeds.groupby(\"supersegment_id\")[\"dummy_eta_freeflow\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7f442016-cd02-41f0-aa58-ad4a2c9d08c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_segment_counts = supersegments_edges_speeds.groupby(\"supersegment_id\")[\"x\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f61919bb-615f-494a-928f-b88a0d4f0e68",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_lengths = supersegments_edges_speeds.groupby(\"supersegment_id\")[\"length_meters\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ec53486a-dc9d-45b8-a163-bcbc4bcd28f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_features = pd.DataFrame(index = supersegment_dummy_eta.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "74ee6ce8-8105-4880-a097-d62893e85903",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 114987/114987 [00:01<00:00, 58141.22it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>supersegment_id</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>edge_int</th>\n",
       "      <th>parsed_maxspeed</th>\n",
       "      <th>speed_kph</th>\n",
       "      <th>importance</th>\n",
       "      <th>highway</th>\n",
       "      <th>oneway</th>\n",
       "      <th>lanes</th>\n",
       "      <th>tunnel</th>\n",
       "      <th>length_meters</th>\n",
       "      <th>counter_distance</th>\n",
       "      <th>volume_class</th>\n",
       "      <th>median_speed_kph</th>\n",
       "      <th>free_flow_kph</th>\n",
       "      <th>dummy_eta</th>\n",
       "      <th>dummy_eta_freeflow</th>\n",
       "      <th>lane_length</th>\n",
       "      <th>avg_lanes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-0.292401</td>\n",
       "      <td>51.528028</td>\n",
       "      <td>130710</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>21.874808</td>\n",
       "      <td>0</td>\n",
       "      <td>4.186018</td>\n",
       "      <td>36.480361</td>\n",
       "      <td>39.254902</td>\n",
       "      <td>2.158677</td>\n",
       "      <td>2.006101</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>84</td>\n",
       "      <td>-0.342000</td>\n",
       "      <td>51.535227</td>\n",
       "      <td>130710</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>21.874808</td>\n",
       "      <td>0</td>\n",
       "      <td>4.186018</td>\n",
       "      <td>36.480361</td>\n",
       "      <td>39.254902</td>\n",
       "      <td>2.158677</td>\n",
       "      <td>2.006101</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>239</td>\n",
       "      <td>-0.292251</td>\n",
       "      <td>51.526640</td>\n",
       "      <td>130710</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>21.874808</td>\n",
       "      <td>0</td>\n",
       "      <td>4.186018</td>\n",
       "      <td>36.480361</td>\n",
       "      <td>39.254902</td>\n",
       "      <td>2.158677</td>\n",
       "      <td>2.006101</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>646</th>\n",
       "      <td>646</td>\n",
       "      <td>-0.351475</td>\n",
       "      <td>51.529245</td>\n",
       "      <td>130710</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>21.874808</td>\n",
       "      <td>0</td>\n",
       "      <td>4.186018</td>\n",
       "      <td>36.480361</td>\n",
       "      <td>39.254902</td>\n",
       "      <td>2.158677</td>\n",
       "      <td>2.006101</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772</th>\n",
       "      <td>772</td>\n",
       "      <td>-0.352310</td>\n",
       "      <td>51.527485</td>\n",
       "      <td>130710</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>21.874808</td>\n",
       "      <td>0</td>\n",
       "      <td>4.186018</td>\n",
       "      <td>36.480361</td>\n",
       "      <td>39.254902</td>\n",
       "      <td>2.158677</td>\n",
       "      <td>2.006101</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>3974</td>\n",
       "      <td>-0.081726</td>\n",
       "      <td>51.364077</td>\n",
       "      <td>22137</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>134.444440</td>\n",
       "      <td>1</td>\n",
       "      <td>3.960736</td>\n",
       "      <td>25.342590</td>\n",
       "      <td>35.882353</td>\n",
       "      <td>19.098285</td>\n",
       "      <td>13.488524</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>3974</td>\n",
       "      <td>-0.081726</td>\n",
       "      <td>51.364077</td>\n",
       "      <td>82289</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>71.554435</td>\n",
       "      <td>2</td>\n",
       "      <td>2.889417</td>\n",
       "      <td>35.988074</td>\n",
       "      <td>40.941176</td>\n",
       "      <td>7.157815</td>\n",
       "      <td>6.291855</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>3974</td>\n",
       "      <td>-0.081726</td>\n",
       "      <td>51.364077</td>\n",
       "      <td>82292</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>80.193630</td>\n",
       "      <td>3</td>\n",
       "      <td>4.118453</td>\n",
       "      <td>24.747924</td>\n",
       "      <td>36.705882</td>\n",
       "      <td>11.665507</td>\n",
       "      <td>7.865144</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>3974</td>\n",
       "      <td>-0.081726</td>\n",
       "      <td>51.364077</td>\n",
       "      <td>113346</td>\n",
       "      <td>48.3</td>\n",
       "      <td>48.3</td>\n",
       "      <td>3</td>\n",
       "      <td>primary</td>\n",
       "      <td>True</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>76.955057</td>\n",
       "      <td>2</td>\n",
       "      <td>4.099010</td>\n",
       "      <td>21.564692</td>\n",
       "      <td>36.705882</td>\n",
       "      <td>12.846842</td>\n",
       "      <td>7.547515</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3990</th>\n",
       "      <td>3990</td>\n",
       "      <td>-0.219238</td>\n",
       "      <td>51.306052</td>\n",
       "      <td>10277</td>\n",
       "      <td>51.4</td>\n",
       "      <td>51.4</td>\n",
       "      <td>4</td>\n",
       "      <td>trunk</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>39.071995</td>\n",
       "      <td>16</td>\n",
       "      <td>3.609477</td>\n",
       "      <td>38.438233</td>\n",
       "      <td>44.627451</td>\n",
       "      <td>3.659356</td>\n",
       "      <td>3.151853</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>114987 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      supersegment_id         x          y  edge_int  parsed_maxspeed  \\\n",
       "0                   0 -0.292401  51.528028    130710             48.3   \n",
       "84                 84 -0.342000  51.535227    130710             48.3   \n",
       "239               239 -0.292251  51.526640    130710             48.3   \n",
       "646               646 -0.351475  51.529245    130710             48.3   \n",
       "772               772 -0.352310  51.527485    130710             48.3   \n",
       "...               ...       ...        ...       ...              ...   \n",
       "3974             3974 -0.081726  51.364077     22137             48.3   \n",
       "3974             3974 -0.081726  51.364077     82289             48.3   \n",
       "3974             3974 -0.081726  51.364077     82292             48.3   \n",
       "3974             3974 -0.081726  51.364077    113346             48.3   \n",
       "3990             3990 -0.219238  51.306052     10277             51.4   \n",
       "\n",
       "      speed_kph  importance  highway  oneway lanes tunnel  length_meters  \\\n",
       "0          48.3           3  primary    True                   21.874808   \n",
       "84         48.3           3  primary    True                   21.874808   \n",
       "239        48.3           3  primary    True                   21.874808   \n",
       "646        48.3           3  primary    True                   21.874808   \n",
       "772        48.3           3  primary    True                   21.874808   \n",
       "...         ...         ...      ...     ...   ...    ...            ...   \n",
       "3974       48.3           3  primary   False                  134.444440   \n",
       "3974       48.3           3  primary   False                   71.554435   \n",
       "3974       48.3           3  primary   False                   80.193630   \n",
       "3974       48.3           3  primary    True                   76.955057   \n",
       "3990       51.4           4    trunk    True     3             39.071995   \n",
       "\n",
       "      counter_distance  volume_class  median_speed_kph  free_flow_kph  \\\n",
       "0                    0      4.186018         36.480361      39.254902   \n",
       "84                   0      4.186018         36.480361      39.254902   \n",
       "239                  0      4.186018         36.480361      39.254902   \n",
       "646                  0      4.186018         36.480361      39.254902   \n",
       "772                  0      4.186018         36.480361      39.254902   \n",
       "...                ...           ...               ...            ...   \n",
       "3974                 1      3.960736         25.342590      35.882353   \n",
       "3974                 2      2.889417         35.988074      40.941176   \n",
       "3974                 3      4.118453         24.747924      36.705882   \n",
       "3974                 2      4.099010         21.564692      36.705882   \n",
       "3990                16      3.609477         38.438233      44.627451   \n",
       "\n",
       "      dummy_eta  dummy_eta_freeflow  lane_length  avg_lanes  \n",
       "0      2.158677            2.006101            1        2.0  \n",
       "84     2.158677            2.006101            1        2.0  \n",
       "239    2.158677            2.006101            1        2.0  \n",
       "646    2.158677            2.006101            1        2.0  \n",
       "772    2.158677            2.006101            1        2.0  \n",
       "...         ...                 ...          ...        ...  \n",
       "3974  19.098285           13.488524            1        2.0  \n",
       "3974   7.157815            6.291855            1        2.0  \n",
       "3974  11.665507            7.865144            1        2.0  \n",
       "3974  12.846842            7.547515            1        2.0  \n",
       "3990   3.659356            3.151853            1        3.0  \n",
       "\n",
       "[114987 rows x 20 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def unpack_lanes(data):\n",
    "    lane_length = []\n",
    "    avg_lanes = []\n",
    "    for l in tqdm(data[\"lanes\"]):\n",
    "        if l:\n",
    "            try:\n",
    "                lanes = eval(l)\n",
    "                if isinstance(lanes, int):\n",
    "                    lanes = [lanes]\n",
    "                if isinstance(lanes, float):\n",
    "                    lanes = [lanes]\n",
    "                elif isinstance(lanes, list):\n",
    "                    lanes = [int(l) for l in lanes]\n",
    "            except:\n",
    "                lanes = [2]\n",
    "        else:\n",
    "            lanes = [2]\n",
    "        lane_length.append(len(lanes))\n",
    "        avg_lanes.append(np.mean(lanes))\n",
    "        \n",
    "    data[\"lane_length\"] = lane_length\n",
    "    data[\"avg_lanes\"] = avg_lanes\n",
    "    \n",
    "    return data\n",
    "\n",
    "unpack_lanes(supersegments_edges_speeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3b0f711e-e618-45c4-8908-27916baffb3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegment_lanes = supersegments_edges_speeds.groupby(\"supersegment_id\")[\"avg_lanes\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ac698df7-1827-418b-a08a-254d675be4e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_features[\"dummy_eta\"] = supersegment_dummy_eta\n",
    "supersegments_features[\"dummy_eta_freeflow\"] = supersegment_dummy_eta_freeflow\n",
    "supersegments_features[\"segment_count\"] = supersegment_segment_counts\n",
    "supersegments_features[\"length\"] = supersegment_lengths\n",
    "supersegments_features[\"lanes\"] = supersegment_lanes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7631d28e-3d93-42fe-b8ff-4119b3351068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dummy_eta</th>\n",
       "      <th>dummy_eta_freeflow</th>\n",
       "      <th>segment_count</th>\n",
       "      <th>length</th>\n",
       "      <th>lanes</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>supersegment_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>476.998710</td>\n",
       "      <td>372.944764</td>\n",
       "      <td>34</td>\n",
       "      <td>5729.235729</td>\n",
       "      <td>2.965686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>402.873689</td>\n",
       "      <td>263.324157</td>\n",
       "      <td>49</td>\n",
       "      <td>2660.969896</td>\n",
       "      <td>2.887755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>242.885249</td>\n",
       "      <td>177.649159</td>\n",
       "      <td>25</td>\n",
       "      <td>1653.841867</td>\n",
       "      <td>2.360000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>210.902577</td>\n",
       "      <td>148.598402</td>\n",
       "      <td>14</td>\n",
       "      <td>1112.990334</td>\n",
       "      <td>2.642857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>66.842443</td>\n",
       "      <td>39.371535</td>\n",
       "      <td>5</td>\n",
       "      <td>600.552555</td>\n",
       "      <td>2.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4007</th>\n",
       "      <td>375.102890</td>\n",
       "      <td>233.569569</td>\n",
       "      <td>27</td>\n",
       "      <td>2382.413855</td>\n",
       "      <td>2.018519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4008</th>\n",
       "      <td>105.165501</td>\n",
       "      <td>89.222790</td>\n",
       "      <td>7</td>\n",
       "      <td>1463.985366</td>\n",
       "      <td>2.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4009</th>\n",
       "      <td>289.957470</td>\n",
       "      <td>225.483505</td>\n",
       "      <td>16</td>\n",
       "      <td>1659.607841</td>\n",
       "      <td>2.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4010</th>\n",
       "      <td>361.711821</td>\n",
       "      <td>256.445335</td>\n",
       "      <td>40</td>\n",
       "      <td>2802.584089</td>\n",
       "      <td>2.025000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011</th>\n",
       "      <td>270.786715</td>\n",
       "      <td>206.968093</td>\n",
       "      <td>19</td>\n",
       "      <td>3045.283399</td>\n",
       "      <td>2.210526</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4012 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  dummy_eta  dummy_eta_freeflow  segment_count       length  \\\n",
       "supersegment_id                                                               \n",
       "0                476.998710          372.944764             34  5729.235729   \n",
       "1                402.873689          263.324157             49  2660.969896   \n",
       "2                242.885249          177.649159             25  1653.841867   \n",
       "3                210.902577          148.598402             14  1112.990334   \n",
       "4                 66.842443           39.371535              5   600.552555   \n",
       "...                     ...                 ...            ...          ...   \n",
       "4007             375.102890          233.569569             27  2382.413855   \n",
       "4008             105.165501           89.222790              7  1463.985366   \n",
       "4009             289.957470          225.483505             16  1659.607841   \n",
       "4010             361.711821          256.445335             40  2802.584089   \n",
       "4011             270.786715          206.968093             19  3045.283399   \n",
       "\n",
       "                    lanes  \n",
       "supersegment_id            \n",
       "0                2.965686  \n",
       "1                2.887755  \n",
       "2                2.360000  \n",
       "3                2.642857  \n",
       "4                2.200000  \n",
       "...                   ...  \n",
       "4007             2.018519  \n",
       "4008             2.714286  \n",
       "4009             2.312500  \n",
       "4010             2.025000  \n",
       "4011             2.210526  \n",
       "\n",
       "[4012 rows x 5 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "supersegments_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8e2bea00-6dcd-46c8-bc4d-196f3377e00f",
   "metadata": {},
   "outputs": [],
   "source": [
    "supersegments_features.to_parquet(data_dir / \"traffic\" / city_name / \"ss_speeds.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d7211d8-25b9-49fd-a27d-b73f5336da96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c611303-2516-4007-bddd-0b8088c448a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "281b1006-90ea-4623-960e-f4811372abc8",
   "metadata": {},
   "source": [
    "<h3> Tests below: check that all artifacts contain the most up to date columns </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0c97a816-9592-4693-b10a-cb19c67754a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = Path(\"/Users/andrei/Desktop/data4cast/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "1902092a-7393-4d6c-911b-4bf7653192db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['dummy_eta', 'dummy_eta_freeflow', 'segment_count', 'length', 'lanes'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "city_name = \"london\"\n",
    "supersegment_speed_features = pd.read_parquet(data_dir / \"traffic\" / city_name / \"ss_speeds.parquet\")\n",
    "print(supersegment_speed_features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "164189b1-3bc0-4397-bdc5-7b15312b8644",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['dummy_eta', 'dummy_eta_freeflow', 'segment_count', 'length', 'lanes'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "city_name = \"melbourne\"\n",
    "supersegment_speed_features = pd.read_parquet(data_dir / \"traffic\" / city_name / \"ss_speeds.parquet\")\n",
    "print(supersegment_speed_features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "db18d987-fdd4-446d-a8d7-1d0f909f85ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['dummy_eta', 'dummy_eta_freeflow', 'segment_count', 'length', 'lanes'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "city_name = \"madrid\"\n",
    "supersegment_speed_features = pd.read_parquet(data_dir / \"traffic\" / city_name / \"ss_speeds.parquet\")\n",
    "print(supersegment_speed_features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b25a54a-9ba2-4c08-be70-6a7f6fa088b9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:t4c22] *",
   "language": "python",
   "name": "conda-env-t4c22-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}